123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026 |
- package services
- import (
- "archive/tar"
- "bytes"
- "context"
- "crypto/rand"
- "encoding/base64"
- "encoding/json"
- "fmt"
- "io"
- "os"
- "os/exec"
- "path/filepath"
- "strings"
- "time"
- "git.linuxforward.com/byop/byop-engine/clients"
- "git.linuxforward.com/byop/byop-engine/dbstore"
- "git.linuxforward.com/byop/byop-engine/models"
- "github.com/sirupsen/logrus"
- "github.com/docker/docker/api/types"
- "github.com/docker/docker/api/types/container"
- "github.com/docker/docker/api/types/filters"
- "github.com/docker/docker/api/types/image"
- "github.com/docker/docker/api/types/registry"
- docker "github.com/docker/docker/client"
- )
- // PreviewService defines the interface for preview services
- type PreviewService interface {
- CreatePreview(ctx context.Context, appId int) (*models.Preview, error)
- DeletePreview(ctx context.Context, appID int) error
- StopPreview(ctx context.Context, previewID int) error
- Close(ctx context.Context) // Updated signature to include context
- }
- // PreviewCommon contains shared functionality for preview services
- type PreviewCommon struct {
- store *dbstore.SQLiteStore
- entry *logrus.Entry
- dockerClient *docker.Client
- registryClient clients.RegistryClient
- registryURL string
- registryUser string
- registryPass string
- }
- // NewPreviewCommon creates a new PreviewCommon instance
- func NewPreviewCommon(store *dbstore.SQLiteStore, registryClient clients.RegistryClient, registryURL, registryUser, registryPass string) *PreviewCommon {
- dockerClient, err := docker.NewClientWithOpts(docker.FromEnv, docker.WithAPIVersionNegotiation())
- if err != nil {
- logrus.WithError(err).Fatal("Failed to create Docker client")
- }
- return &PreviewCommon{
- store: store,
- entry: logrus.WithField("service", "PreviewCommon"),
- dockerClient: dockerClient,
- registryClient: registryClient,
- registryURL: registryURL,
- registryUser: registryUser,
- registryPass: registryPass,
- }
- }
- // Close cleans up the Docker client connection
- func (pc *PreviewCommon) Close() {
- // Clean up BYOP preview images
- pc.CleanupPreviewImages(context.Background())
- // Clean up preview database state
- pc.CleanupPreviewState(context.Background())
- // Close the Docker client connection
- if pc.dockerClient != nil {
- if err := pc.dockerClient.Close(); err != nil {
- pc.entry.WithError(err).Error("Failed to close Docker client")
- } else {
- pc.entry.Info("Docker client connection closed")
- }
- }
- }
- // GetDockerClient returns the Docker client
- func (pc *PreviewCommon) GetDockerClient() *docker.Client {
- return pc.dockerClient
- }
- // GetStore returns the database store
- func (pc *PreviewCommon) GetStore() *dbstore.SQLiteStore {
- return pc.store
- }
- // GetLogger returns the logger
- func (pc *PreviewCommon) GetLogger() *logrus.Entry {
- return pc.entry
- }
- // GeneratePreviewID generates an 8-character random hex UUID for preview URLs
- func (pc *PreviewCommon) GeneratePreviewID() string {
- bytes := make([]byte, 4) // 4 bytes = 8 hex chars
- if _, err := rand.Read(bytes); err != nil {
- // Fallback to timestamp-based ID if crypto/rand fails
- return fmt.Sprintf("%08x", time.Now().Unix()%0xFFFFFFFF)
- }
- // Convert each byte directly to hex to ensure we get truly random looking IDs
- return fmt.Sprintf("%02x%02x%02x%02x", bytes[0], bytes[1], bytes[2], bytes[3])
- }
- // CloneRepository clones a git repository to a target directory
- func (pc *PreviewCommon) CloneRepository(ctx context.Context, repoURL, branch, targetDir string) error {
- if err := os.MkdirAll(targetDir, 0755); err != nil {
- return models.NewErrInternalServer(fmt.Sprintf("failed to create target directory %s", targetDir), err)
- }
- if branch == "" {
- branch = "main"
- }
- cmd := exec.CommandContext(ctx, "git", "clone", "--depth", "1", "--branch", branch, repoURL, targetDir)
- if err := cmd.Run(); err != nil {
- // Try with master branch if main fails
- if branch == "main" {
- cmd = exec.CommandContext(ctx, "git", "clone", "--depth", "1", "--branch", "master", repoURL, targetDir)
- if err := cmd.Run(); err != nil {
- return models.NewErrInternalServer(fmt.Sprintf("failed to clone repository (tried main and master branches): %s", repoURL), err)
- }
- } else {
- return models.NewErrInternalServer(fmt.Sprintf("failed to clone repository %s on branch %s", repoURL, branch), err)
- }
- }
- return nil
- }
- // CreateBuildContext creates a tar archive of the build context
- func (pc *PreviewCommon) CreateBuildContext(ctx context.Context, contextDir string) (io.ReadCloser, error) {
- var buf bytes.Buffer
- tw := tar.NewWriter(&buf)
- defer tw.Close()
- // Common ignore patterns for Git repositories
- ignorePatterns := []string{
- ".git",
- ".gitignore",
- "node_modules",
- ".next",
- "dist",
- "build",
- "target",
- "__pycache__",
- "*.pyc",
- ".DS_Store",
- "Thumbs.db",
- "*.log",
- "*.tmp",
- "*.swp",
- ".env",
- ".vscode",
- ".idea",
- "playwright",
- "cypress",
- "coverage",
- "*.test.js",
- "*.spec.js",
- "*.test.ts",
- "*.spec.ts",
- "test",
- "tests",
- "__tests__",
- "snapshots",
- "*.png",
- "*.jpg",
- "*.jpeg",
- "*.gif",
- "*.bmp",
- "*.svg",
- "*.ico",
- "*.zip",
- "*.tar.gz",
- "*.tar",
- "*.gz",
- "README.md",
- "readme.md",
- "CHANGELOG.md",
- "LICENSE",
- "CONTRIBUTING.md",
- "*.md",
- "docs",
- "documentation",
- }
- err := filepath.Walk(contextDir, func(file string, fi os.FileInfo, err error) error {
- if err != nil {
- return err
- }
- // Get relative path
- relPath, err := filepath.Rel(contextDir, file)
- if err != nil {
- return err
- }
- // Skip if matches ignore patterns
- for _, pattern := range ignorePatterns {
- if matched, _ := filepath.Match(pattern, fi.Name()); matched {
- if fi.IsDir() {
- return filepath.SkipDir
- }
- return nil
- }
- if strings.Contains(relPath, pattern) {
- if fi.IsDir() {
- return filepath.SkipDir
- }
- return nil
- }
- }
- // Skip very large files (> 100MB)
- if !fi.IsDir() && fi.Size() > 100*1024*1024 {
- pc.entry.WithField("file", relPath).WithField("size", fi.Size()).Warn("Skipping large file in build context")
- return nil
- }
- // Skip files with very long paths (> 200 chars)
- if len(relPath) > 200 {
- pc.entry.WithField("file", relPath).WithField("length", len(relPath)).Warn("Skipping file with very long path")
- return nil
- }
- // Create tar header
- header, err := tar.FileInfoHeader(fi, fi.Name())
- if err != nil {
- return err
- }
- // Update the name to be relative to the context directory
- header.Name = filepath.ToSlash(relPath)
- // Ensure header name is not too long for tar format
- if len(header.Name) > 155 {
- pc.entry.WithField("file", header.Name).WithField("length", len(header.Name)).Warn("Skipping file with tar-incompatible long name")
- return nil
- }
- // Write header
- if err := tw.WriteHeader(header); err != nil {
- return fmt.Errorf("failed to write tar header for %s: %v", relPath, err)
- }
- // If it's a file, write its content
- if !fi.IsDir() {
- data, err := os.Open(file)
- if err != nil {
- return fmt.Errorf("failed to open file %s: %v", relPath, err)
- }
- defer data.Close()
- // Use limited reader to prevent issues with very large files
- limitedReader := io.LimitReader(data, 100*1024*1024) // 100MB limit
- written, err := io.Copy(tw, limitedReader)
- if err != nil {
- pc.entry.WithField("file", relPath).WithField("written_bytes", written).Warnf("Failed to copy file to tar, skipping: %v", err)
- // Don't return error, just skip this file
- return nil
- }
- }
- return nil
- })
- if err != nil {
- return nil, err
- }
- return io.NopCloser(&buf), nil
- }
- // createDockerIgnore creates a .dockerignore file in the specified directory
- func (pc *PreviewCommon) createDockerIgnore(ctx context.Context, contextDir string) {
- dockerignoreContent := `# Auto-generated by BYOP Engine
- .git
- .gitignore
- node_modules
- .next
- dist
- build
- target
- __pycache__
- *.pyc
- .DS_Store
- Thumbs.db
- *.log
- *.tmp
- *.swp
- .env
- .vscode
- .idea
- playwright
- cypress
- coverage
- test
- tests
- __tests__
- snapshots
- *.test.js
- *.spec.js
- *.test.ts
- *.spec.ts
- *.png
- *.jpg
- *.jpeg
- *.gif
- *.bmp
- *.svg
- *.ico
- *.zip
- *.tar.gz
- *.tar
- *.gz
- README.md
- readme.md
- CHANGELOG.md
- LICENSE
- CONTRIBUTING.md
- *.md
- docs
- documentation
- `
- dockerignorePath := filepath.Join(contextDir, ".dockerignore")
- if err := os.WriteFile(dockerignorePath, []byte(dockerignoreContent), 0644); err != nil {
- pc.entry.WithField("path", dockerignorePath).Warnf("Failed to create .dockerignore file: %v", err)
- } else {
- pc.entry.WithField("path", dockerignorePath).Debug("Created .dockerignore file")
- }
- }
- // validateAndFixDockerfile checks for unsupported Dockerfile syntax and fixes common issues
- func (pc *PreviewCommon) validateAndFixDockerfile(ctx context.Context, contextDir string) error {
- dockerfilePath := filepath.Join(contextDir, "Dockerfile")
- // Check if Dockerfile exists
- if _, err := os.Stat(dockerfilePath); os.IsNotExist(err) {
- return fmt.Errorf("dockerfile not found in repository")
- }
- // Read the Dockerfile
- content, err := os.ReadFile(dockerfilePath)
- if err != nil {
- return fmt.Errorf("failed to read Dockerfile: %v", err)
- }
- originalContent := string(content)
- modifiedContent := originalContent
- modified := false
- // Fix common issues
- lines := strings.Split(originalContent, "\n")
- var fixedLines []string
- for i, line := range lines {
- trimmedLine := strings.TrimSpace(line)
- // Check for unsupported --exclude flag in COPY or ADD commands
- if strings.HasPrefix(trimmedLine, "COPY") || strings.HasPrefix(trimmedLine, "ADD") {
- if strings.Contains(trimmedLine, "--exclude") {
- pc.entry.WithField("line", i+1).Warn("Found unsupported --exclude flag in Dockerfile, removing it")
- // Remove --exclude flag and its arguments
- parts := strings.Fields(trimmedLine)
- var cleanedParts []string
- skipNext := false
- for _, part := range parts {
- if skipNext {
- skipNext = false
- continue
- }
- if strings.HasPrefix(part, "--exclude") {
- if strings.Contains(part, "=") {
- // --exclude=pattern format
- continue
- } else {
- // --exclude pattern format
- skipNext = true
- continue
- }
- }
- cleanedParts = append(cleanedParts, part)
- }
- fixedLine := strings.Join(cleanedParts, " ")
- fixedLines = append(fixedLines, fixedLine)
- modified = true
- pc.entry.WithField("original", trimmedLine).WithField("fixed", fixedLine).Info("Fixed Dockerfile line")
- } else {
- fixedLines = append(fixedLines, line)
- }
- } else {
- fixedLines = append(fixedLines, line)
- }
- }
- // Write back the fixed Dockerfile if modified
- if modified {
- modifiedContent = strings.Join(fixedLines, "\n")
- if err := os.WriteFile(dockerfilePath, []byte(modifiedContent), 0644); err != nil {
- return fmt.Errorf("failed to write fixed Dockerfile: %v", err)
- }
- pc.entry.WithField("path", dockerfilePath).Info("Fixed Dockerfile syntax issues")
- }
- return nil
- }
- // isDockerfilePresent checks if a Dockerfile exists in the repository directory
- func (pc *PreviewCommon) isDockerfilePresent(tempDir string) (bool, error) {
- // Check for common Dockerfile names
- dockerfileNames := []string{"Dockerfile", "dockerfile", "Dockerfile.prod", "Dockerfile.production"}
- for _, name := range dockerfileNames {
- dockerfilePath := filepath.Join(tempDir, name)
- if _, err := os.Stat(dockerfilePath); err == nil {
- pc.entry.WithField("dockerfile_path", dockerfilePath).Debug("Found Dockerfile")
- return true, nil
- }
- }
- pc.entry.WithField("temp_dir", tempDir).Debug("No Dockerfile found")
- return false, nil
- }
- // BuildComponentImages builds Docker images for components
- // It first checks for pre-built images in the registry before rebuilding from source
- func (pc *PreviewCommon) BuildComponentImages(ctx context.Context, components []models.Component) ([]string, string, error) {
- var imageNames []string
- var allLogs strings.Builder
- for _, component := range components {
- pc.entry.WithField("component_id", component.ID).WithField("status", component.Status).Info("Processing component for preview")
- // Generate local image name for preview
- imageName := fmt.Sprintf("byop-preview-%s:%d", component.Name, component.ID)
- // Check if component has pre-built image information
- if component.CurrentImageURI != "" && component.CurrentImageTag != "" {
- pc.entry.WithField("component_id", component.ID).WithField("image_uri", component.CurrentImageURI).Info("Component has pre-built image, checking registry")
- allLogs.WriteString(fmt.Sprintf("Component %d has pre-built image %s, checking availability\n", component.ID, component.CurrentImageURI))
- // Check if the pre-built image exists in the registry
- if pc.registryClient != nil && pc.registryURL != "" {
- exists, err := pc.registryClient.CheckImageExists(ctx, component.CurrentImageURI, pc.registryURL, pc.registryUser, pc.registryPass)
- if err != nil {
- pc.entry.WithField("component_id", component.ID).WithError(err).Warn("Failed to check if pre-built image exists, falling back to rebuild")
- allLogs.WriteString(fmt.Sprintf("Failed to check registry image for component %d: %v, rebuilding from source\n", component.ID, err))
- } else if exists {
- // Pull the pre-built image from registry to local Docker
- if err := pc.pullPreBuiltImage(ctx, component.CurrentImageURI, imageName); err != nil {
- pc.entry.WithField("component_id", component.ID).WithError(err).Warn("Failed to pull pre-built image, falling back to rebuild")
- allLogs.WriteString(fmt.Sprintf("Failed to pull pre-built image for component %d: %v, rebuilding from source\n", component.ID, err))
- } else {
- pc.entry.WithField("component_id", component.ID).WithField("image_name", imageName).Info("Successfully used pre-built image")
- allLogs.WriteString(fmt.Sprintf("Successfully pulled and tagged pre-built image for component %d as %s\n", component.ID, imageName))
- imageNames = append(imageNames, imageName)
- continue // Skip to next component
- }
- } else {
- pc.entry.WithField("component_id", component.ID).Info("Pre-built image not found in registry, rebuilding from source")
- allLogs.WriteString(fmt.Sprintf("Pre-built image for component %d not found in registry, rebuilding from source\n", component.ID))
- }
- } else {
- pc.entry.WithField("component_id", component.ID).Warn("Registry client not configured, cannot check pre-built images")
- allLogs.WriteString(fmt.Sprintf("Registry not configured for component %d, rebuilding from source\n", component.ID))
- }
- } else {
- pc.entry.WithField("component_id", component.ID).Info("Component has no pre-built image information, building from source")
- allLogs.WriteString(fmt.Sprintf("Component %d has no pre-built image, building from source\n", component.ID))
- }
- // Fallback: Build from source code
- pc.entry.WithField("component_id", component.ID).Info("Building Docker image from source")
- allLogs.WriteString(fmt.Sprintf("Building component %d from source\n", component.ID))
- // Create temp directory for this component
- tempDir := filepath.Join(os.TempDir(), fmt.Sprintf("byop-preview-%d-%d", component.ID, time.Now().Unix()))
- defer os.RemoveAll(tempDir)
- // Clone repository
- if err := pc.CloneRepository(ctx, component.Repository, component.Branch, tempDir); err != nil {
- allLogs.WriteString(fmt.Sprintf("Failed to clone %s: %v\n", component.Repository, err))
- return nil, allLogs.String(), err
- }
- // Special handling for components with existing Dockerfiles (status "valid")
- if component.Status == "valid" {
- pc.entry.WithField("component_id", component.ID).Info("Component has existing Dockerfile, building directly")
- allLogs.WriteString(fmt.Sprintf("Component %d has existing Dockerfile, building directly\n", component.ID))
- // For components with existing Dockerfiles, just use the Dockerfile as-is
- // No need to validate/fix or create .dockerignore since they should work as-is
- } else {
- // For components without existing Dockerfiles (generated via LLB), apply fixes
- pc.entry.WithField("component_id", component.ID).Info("Component using generated Dockerfile, applying fixes")
- // Create .dockerignore file to exclude unnecessary files
- pc.createDockerIgnore(ctx, tempDir)
- // Check and fix Dockerfile if needed
- if err := pc.validateAndFixDockerfile(ctx, tempDir); err != nil {
- allLogs.WriteString(fmt.Sprintf("Failed to validate Dockerfile for component %d: %v\n", component.ID, err))
- return nil, allLogs.String(), err
- }
- }
- // Create a tar archive of the build context
- pc.entry.WithField("component_id", component.ID).Info("Creating build context tar archive")
- tarReader, err := pc.CreateBuildContext(ctx, tempDir)
- if err != nil {
- errMsg := fmt.Sprintf("Failed to create build context for %s: %v", imageName, err)
- pc.entry.WithField("component_id", component.ID).Error(errMsg)
- allLogs.WriteString(errMsg + "\n")
- return nil, allLogs.String(), err
- }
- defer tarReader.Close()
- pc.entry.WithField("component_id", component.ID).WithField("image_name", imageName).Info("Starting Docker image build")
- buildResponse, err := pc.dockerClient.ImageBuild(ctx, tarReader, types.ImageBuildOptions{
- Tags: []string{imageName},
- Dockerfile: "Dockerfile",
- Remove: true,
- ForceRemove: true,
- })
- if err != nil {
- errMsg := fmt.Sprintf("Failed to start build for %s: %v", imageName, err)
- pc.entry.WithField("component_id", component.ID).Error(errMsg)
- allLogs.WriteString(errMsg + "\n")
- return nil, allLogs.String(), err
- }
- defer buildResponse.Body.Close()
- // Read and parse build output properly
- buildOutput, err := io.ReadAll(buildResponse.Body)
- if err != nil {
- allLogs.WriteString(fmt.Sprintf("Failed to read build output for %s: %v\n", imageName, err))
- return nil, allLogs.String(), err
- }
- buildOutputStr := string(buildOutput)
- allLogs.WriteString(fmt.Sprintf("Building %s:\n%s\n", imageName, buildOutputStr))
- // Check for Docker build errors in JSON output
- buildSuccess := false
- buildErrorFound := false
- // Parse each line of JSON output
- lines := strings.Split(buildOutputStr, "\n")
- for _, line := range lines {
- line = strings.TrimSpace(line)
- if line == "" {
- continue
- }
- // Look for success indicators
- if strings.Contains(line, `"stream":"Successfully built`) ||
- strings.Contains(line, `"stream":"Successfully tagged`) {
- buildSuccess = true
- }
- // Look for error indicators
- if strings.Contains(line, `"error"`) ||
- strings.Contains(line, `"errorDetail"`) ||
- strings.Contains(line, `"stream":"ERROR`) ||
- strings.Contains(line, `"stream":"The command"`) && strings.Contains(line, "returned a non-zero code") {
- buildErrorFound = true
- allLogs.WriteString(fmt.Sprintf("Build error detected in line: %s\n", line))
- }
- }
- if buildErrorFound {
- allLogs.WriteString(fmt.Sprintf("Build failed for %s: errors found in build output\n", imageName))
- return nil, allLogs.String(), fmt.Errorf("docker build failed for %s: check build logs", imageName)
- }
- if !buildSuccess {
- allLogs.WriteString(fmt.Sprintf("Build failed for %s: no success indicators found in build output\n", imageName))
- return nil, allLogs.String(), fmt.Errorf("docker build failed for %s: build did not complete successfully", imageName)
- }
- // Verify the image exists and is properly tagged
- _, err = pc.dockerClient.ImageInspect(ctx, imageName)
- if err != nil {
- allLogs.WriteString(fmt.Sprintf("Build verification failed for %s: image not found after build - %v\n", imageName, err))
- return nil, allLogs.String(), fmt.Errorf("failed to build image %s: image not found after build", imageName)
- }
- imageNames = append(imageNames, imageName)
- pc.entry.WithField("component_id", component.ID).WithField("image_name", imageName).Info("Successfully built Docker image")
- }
- return imageNames, allLogs.String(), nil
- }
- // pullPreBuiltImage pulls a pre-built image from the registry and tags it for local use
- func (pc *PreviewCommon) pullPreBuiltImage(ctx context.Context, registryImageURI, localImageName string) error {
- pc.entry.WithField("registry_image", registryImageURI).WithField("local_image", localImageName).Info("Pulling pre-built image from registry")
- // Pull the image from registry
- pullOptions := image.PullOptions{}
- // Add authentication if registry credentials are configured
- if pc.registryUser != "" && pc.registryPass != "" {
- authConfig := registry.AuthConfig{
- Username: pc.registryUser,
- Password: pc.registryPass,
- }
- encodedJSON, err := json.Marshal(authConfig)
- if err != nil {
- return fmt.Errorf("failed to encode registry auth: %w", err)
- }
- pullOptions.RegistryAuth = base64.URLEncoding.EncodeToString(encodedJSON)
- }
- reader, err := pc.dockerClient.ImagePull(ctx, registryImageURI, pullOptions)
- if err != nil {
- return fmt.Errorf("failed to pull image %s: %w", registryImageURI, err)
- }
- defer reader.Close()
- // Read the pull output (similar to build output)
- pullOutput, err := io.ReadAll(reader)
- if err != nil {
- return fmt.Errorf("failed to read pull output: %w", err)
- }
- pc.entry.WithField("pull_output", string(pullOutput)).Debug("Image pull completed")
- // Tag the pulled image with the local preview tag
- err = pc.dockerClient.ImageTag(ctx, registryImageURI, localImageName)
- if err != nil {
- return fmt.Errorf("failed to tag image %s as %s: %w", registryImageURI, localImageName, err)
- }
- // Verify the image is now available locally
- _, err = pc.dockerClient.ImageInspect(ctx, localImageName)
- if err != nil {
- return fmt.Errorf("failed to verify locally tagged image %s: %w", localImageName, err)
- }
- pc.entry.WithField("local_image", localImageName).Info("Successfully pulled and tagged pre-built image")
- return nil
- }
- // GetAppComponents retrieves components for an app
- func (pc *PreviewCommon) GetAppComponents(ctx context.Context, app *models.App) ([]models.Component, error) {
- var components []models.Component
- for _, componentID := range app.Components {
- component, err := pc.store.GetComponentByID(ctx, componentID)
- if err != nil {
- return nil, err
- }
- if component == nil {
- return nil, models.NewErrNotFound(fmt.Sprintf("Component with ID %d not found while fetching app components", componentID), nil)
- }
- components = append(components, *component)
- }
- return components, nil
- }
- // CleanupPreviewImages cleans up BYOP preview Docker images
- func (pc *PreviewCommon) CleanupPreviewImages(ctx context.Context) {
- pc.entry.Info("Cleaning up BYOP preview images...")
- images, err := pc.dockerClient.ImageList(ctx, image.ListOptions{All: true})
- if err != nil {
- pc.entry.WithError(err).Error("Failed to list images for cleanup")
- return
- }
- removedCount := 0
- for _, img := range images {
- // Check if image name contains "byop-preview"
- isPreviewImage := false
- for _, tag := range img.RepoTags {
- if strings.Contains(tag, "byop-preview") {
- isPreviewImage = true
- break
- }
- }
- if !isPreviewImage {
- continue
- }
- // Remove the image
- if _, err := pc.dockerClient.ImageRemove(ctx, img.ID, image.RemoveOptions{
- Force: true,
- PruneChildren: true,
- }); err != nil {
- pc.entry.WithError(err).WithField("image_id", img.ID).Warn("Failed to remove preview image")
- } else {
- removedCount++
- }
- }
- if removedCount > 0 {
- pc.entry.WithField("removed_images", removedCount).Info("Cleaned up BYOP preview images")
- }
- }
- // CleanupByAppID cleans up all BYOP preview containers and images for a specific app ID
- func (pc *PreviewCommon) CleanupByAppID(ctx context.Context, appID int) {
- pc.entry.WithField("app_id", appID).Info("Cleaning up BYOP preview containers...")
- // List all containers
- containers, err := pc.dockerClient.ContainerList(ctx, container.ListOptions{All: true})
- if err != nil {
- pc.entry.WithError(err).Error("Failed to list containers for cleanup")
- return
- }
- for _, ctn := range containers {
- isPreviewContainer := false
- containerName := ""
- // Check if the container is a BYOP preview container
- for key, value := range ctn.Labels {
- if key == "byop.preview" && value == "true" {
- isPreviewContainer = true
- if len(ctn.Names) > 0 {
- containerName = ctn.Names[0]
- }
- break
- }
- }
- if !isPreviewContainer {
- continue
- }
- if ctn.Labels["byop.app.id"] != fmt.Sprintf("%d", appID) {
- continue // Only clean up containers for the specified app ID
- }
- pc.entry.WithField("container_id", ctn.ID).WithField("container_name", containerName).Info("Removing BYOP preview container")
- // Remove the container
- if err := pc.dockerClient.ContainerRemove(ctx, ctn.ID, container.RemoveOptions{
- Force: true,
- }); err != nil {
- pc.entry.WithError(err).WithField("container_id", ctn.ID).Warn("Failed to remove preview container")
- } else {
- pc.entry.WithField("container_id", ctn.ID).Info("Successfully removed preview container")
- }
- }
- }
- // CleanupAllPreviewContainers cleans up all BYOP preview containers
- func (pc *PreviewCommon) CleanupAllPreviewContainers(ctx context.Context) {
- pc.entry.Info("Cleaning up all BYOP preview containers...")
- // Get all containers with filters for BYOP preview containers
- containers, err := pc.dockerClient.ContainerList(ctx, container.ListOptions{
- All: true, // Include stopped containers too
- Filters: filters.NewArgs(
- filters.Arg("label", "byop.preview=true"),
- ),
- })
- if err != nil {
- pc.entry.WithError(err).Error("Failed to list BYOP preview containers")
- // Fallback to name-based filtering if labels don't work
- pc.cleanupByName(ctx)
- return
- }
- if len(containers) == 0 {
- pc.entry.Info("No BYOP preview containers found to cleanup")
- } else {
- pc.entry.WithField("container_count", len(containers)).Info("Found BYOP preview containers to cleanup")
- }
- // Remove BYOP preview containers
- for _, ctn := range containers {
- containerName := "unknown"
- if len(ctn.Names) > 0 {
- containerName = strings.TrimPrefix(ctn.Names[0], "/")
- }
- pc.entry.WithField("container_id", ctn.ID).WithField("container_name", containerName).Info("Removing BYOP preview container")
- // Stop container first if it's running
- if ctn.State == "running" {
- if err := pc.dockerClient.ContainerStop(ctx, ctn.ID, container.StopOptions{}); err != nil {
- pc.entry.WithError(err).WithField("container_id", ctn.ID).Warn("Failed to stop container, will force remove")
- }
- }
- // Remove container
- if err := pc.dockerClient.ContainerRemove(ctx, ctn.ID, container.RemoveOptions{
- Force: true,
- RemoveVolumes: true,
- }); err != nil {
- pc.entry.WithError(err).WithField("container_id", ctn.ID).Error("Failed to remove BYOP preview container")
- } else {
- pc.entry.WithField("container_id", ctn.ID).WithField("container_name", containerName).Info("Successfully removed BYOP preview container")
- }
- }
- }
- // Fallback method to cleanup containers by name pattern
- func (pc *PreviewCommon) cleanupByName(ctx context.Context) {
- pc.entry.Info("Using fallback name-based container cleanup")
- containers, err := pc.dockerClient.ContainerList(ctx, container.ListOptions{All: true})
- if err != nil {
- pc.entry.WithError(err).Error("Failed to list containers for name-based cleanup")
- return
- }
- for _, ctn := range containers {
- // Check if any container name contains "byop-preview"
- isPreviewContainer := false
- containerName := "unknown"
- for _, name := range ctn.Names {
- cleanName := strings.TrimPrefix(name, "/")
- if strings.Contains(cleanName, "byop-preview") || strings.Contains(cleanName, "preview") {
- isPreviewContainer = true
- containerName = cleanName
- break
- }
- }
- if !isPreviewContainer {
- continue
- }
- pc.entry.WithField("container_id", ctn.ID).WithField("container_name", containerName).Info("Removing BYOP preview container (name-based)")
- // Stop and remove
- if ctn.State == "running" {
- pc.dockerClient.ContainerStop(ctx, ctn.ID, container.StopOptions{})
- }
- if err := pc.dockerClient.ContainerRemove(ctx, ctn.ID, container.RemoveOptions{
- Force: true,
- RemoveVolumes: true,
- }); err != nil {
- pc.entry.WithError(err).WithField("container_id", ctn.ID).Error("Failed to remove container")
- }
- }
- }
- // CleanupPreviewState cleans up preview database state - mark all running previews as stopped
- func (pc *PreviewCommon) CleanupPreviewState(ctx context.Context) {
- pc.entry.Info("Cleaning up preview database state...")
- // Get all active previews (building, deploying, running)
- activeStatuses := []string{"building", "deploying", "running"}
- for _, status := range activeStatuses {
- previews, err := pc.store.GetPreviewsByStatus(ctx, status)
- if err != nil {
- pc.entry.WithError(err).WithField("status", status).Error("Failed to get previews by status")
- continue
- }
- for _, preview := range previews {
- pc.entry.WithField("preview_id", preview.ID).WithField("app_id", preview.AppID).WithField("old_status", preview.Status).Info("Marking preview as stopped due to server shutdown")
- // Update preview status to stopped
- if err := pc.store.UpdatePreviewStatus(ctx, preview.ID, "stopped", "Server shutdown - containers may have been stopped"); err != nil {
- pc.entry.WithError(err).WithField("preview_id", preview.ID).Error("Failed to update preview status to stopped")
- }
- // Also update the associated app status back to "ready" if it was in a preview state
- if app, err := pc.store.GetAppByID(ctx, preview.AppID); err == nil && app != nil {
- if app.Status == "building" || app.Status == "deploying" {
- if err := pc.store.UpdateAppStatus(ctx, app.ID, "ready", ""); err != nil {
- pc.entry.WithError(err).WithField("app_id", app.ID).Error("Failed to reset app status to ready")
- } else {
- pc.entry.WithField("app_id", app.ID).Info("Reset app status to ready after preview cleanup")
- }
- }
- }
- }
- if len(previews) > 0 {
- pc.entry.WithField("count", len(previews)).WithField("status", status).Info("Updated preview statuses to stopped")
- }
- }
- pc.entry.Info("Preview database state cleanup completed")
- }
- // GetPreviewImageNames reconstructs the Docker image names used for a preview
- func (pc *PreviewCommon) GetPreviewImageNames(appID int) ([]string, error) {
- // Get app details
- app, err := pc.store.GetAppByID(context.Background(), appID)
- if err != nil {
- return nil, fmt.Errorf("failed to get app by ID %d: %v", appID, err)
- }
- // Get all components for the app
- components, err := pc.GetAppComponents(context.Background(), app)
- if err != nil {
- return nil, fmt.Errorf("failed to get app components: %v", err)
- }
- // Reconstruct image names using the same format as BuildComponentImages
- var imageNames []string
- for _, component := range components {
- imageName := fmt.Sprintf("byop-preview-%s:%d", component.Name, component.ID)
- imageNames = append(imageNames, imageName)
- }
- return imageNames, nil
- }
- // CleanupPreviewImagesForApp cleans up Docker images for a specific app (works for both local and remote)
- func (pc *PreviewCommon) CleanupPreviewImagesForApp(ctx context.Context, appID int, isRemote bool, ipAddress string) error {
- imageNames, err := pc.GetPreviewImageNames(appID)
- if err != nil {
- pc.entry.WithField("app_id", appID).WithError(err).Warn("Failed to get preview image names for cleanup")
- return err
- }
- if isRemote && ipAddress != "" && ipAddress != "127.0.0.1" {
- return pc.cleanupRemoteDockerImages(ctx, ipAddress, imageNames)
- } else {
- return pc.cleanupLocalDockerImages(ctx, imageNames)
- }
- }
- // cleanupLocalDockerImages removes specific Docker images locally
- func (pc *PreviewCommon) cleanupLocalDockerImages(ctx context.Context, imageNames []string) error {
- pc.entry.WithField("image_count", len(imageNames)).Info("Cleaning up specific Docker images locally")
- for _, imageName := range imageNames {
- // Remove the image locally using Docker client
- if _, err := pc.dockerClient.ImageRemove(ctx, imageName, image.RemoveOptions{
- Force: true,
- PruneChildren: true,
- }); err != nil {
- // Log warning but don't fail the cleanup - image might already be removed or in use
- pc.entry.WithField("image_name", imageName).WithError(err).Warn("Failed to remove Docker image locally (this may be normal)")
- } else {
- pc.entry.WithField("image_name", imageName).Info("Successfully removed Docker image locally")
- }
- }
- return nil
- }
- // cleanupRemoteDockerImages removes Docker images from a VPS via SSH
- func (pc *PreviewCommon) cleanupRemoteDockerImages(ctx context.Context, ipAddress string, imageNames []string) error {
- pc.entry.WithField("ip_address", ipAddress).WithField("image_count", len(imageNames)).Info("Cleaning up Docker images on VPS")
- for _, imageName := range imageNames {
- // Remove the image
- rmImageCmd := fmt.Sprintf("docker rmi %s --force", imageName)
- pc.entry.WithField("image_name", imageName).WithField("ip_address", ipAddress).Info("Removing Docker image")
- if err := pc.executeSSHCommand(ctx, ipAddress, rmImageCmd); err != nil {
- // Log warning but don't fail the cleanup - image might already be removed or in use
- pc.entry.WithField("image_name", imageName).WithField("ip_address", ipAddress).WithError(err).Warn("Failed to remove Docker image (this may be normal)")
- } else {
- pc.entry.WithField("image_name", imageName).WithField("ip_address", ipAddress).Info("Successfully removed Docker image")
- }
- // Also remove the tar file if it exists
- tarFileName := strings.ReplaceAll(imageName, ":", "_")
- rmTarCmd := fmt.Sprintf("rm -f /tmp/%s.tar", tarFileName)
- pc.executeSSHCommand(ctx, ipAddress, rmTarCmd) // Ignore errors for tar cleanup
- }
- // Clean up any dangling images
- pc.entry.WithField("ip_address", ipAddress).Info("Cleaning up dangling Docker images")
- danglingCmd := "docker image prune -f"
- if err := pc.executeSSHCommand(ctx, ipAddress, danglingCmd); err != nil {
- pc.entry.WithField("ip_address", ipAddress).WithError(err).Warn("Failed to clean dangling images")
- }
- return nil
- }
- // executeSSHCommand executes a command on a remote VPS via SSH
- func (pc *PreviewCommon) executeSSHCommand(ctx context.Context, ipAddress, command string) error {
- pc.entry.WithField("ip_address", ipAddress).WithField("command", command).Debug("Executing SSH command")
- cmd := exec.CommandContext(ctx, "ssh", "-o", "StrictHostKeyChecking=no", ipAddress, command)
- output, err := cmd.CombinedOutput()
- if err != nil {
- pc.entry.WithField("ip_address", ipAddress).WithField("command", command).WithField("output", string(output)).WithError(err).Error("SSH command failed")
- return models.NewErrInternalServer(fmt.Sprintf("SSH command failed on %s: %s. Output: %s", ipAddress, command, string(output)), err)
- }
- if len(output) > 0 {
- pc.entry.WithField("ip_address", ipAddress).WithField("command", command).WithField("output", string(output)).Debug("SSH command output")
- }
- return nil
- }
- // Database helper methods
- func (pc *PreviewCommon) UpdatePreviewStatus(ctx context.Context, previewID int, status, errorMsg string) {
- if err := pc.store.UpdatePreviewStatus(ctx, previewID, status, errorMsg); err != nil {
- pc.entry.WithField("preview_id", previewID).Errorf("Failed to update preview status: %v", err)
- }
- }
- func (pc *PreviewCommon) UpdatePreviewBuildLogs(ctx context.Context, previewID int, logs string) {
- if err := pc.store.UpdatePreviewBuildLogs(ctx, previewID, logs); err != nil {
- pc.entry.WithField("preview_id", previewID).Errorf("Failed to update preview build logs: %v", err)
- }
- }
- func (pc *PreviewCommon) UpdatePreviewDeployLogs(ctx context.Context, previewID int, logs string) {
- if err := pc.store.UpdatePreviewDeployLogs(ctx, previewID, logs); err != nil {
- pc.entry.WithField("preview_id", previewID).Errorf("Failed to update preview deploy logs: %v", err)
- }
- }
|