package services import ( "context" "encoding/json" "fmt" "os" // Added import "path/filepath" "strings" "time" "git.linuxforward.com/byop/byop-engine/clients" "git.linuxforward.com/byop/byop-engine/dbstore" "git.linuxforward.com/byop/byop-engine/models" git "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing" "github.com/sirupsen/logrus" ) const ( defaultBuildQueueSize = 100 defaultImageTag = "latest" defaultDockerfilePath = "Dockerfile" defaultBuildContext = "." ) // Builder handles the queuing and processing of application build jobs. type Builder struct { store *dbstore.SQLiteStore buildMachineClient clients.BuildMachineClient registryClient clients.RegistryClient maxConcurrentBuild int buildChan chan uint entry *logrus.Entry } // NewBuilderService creates a new Builder service. func NewBuilderService(store *dbstore.SQLiteStore, buildMachineClient clients.BuildMachineClient, registryClient clients.RegistryClient, maxConcurrentBuild int) *Builder { svc := &Builder{ store: store, buildMachineClient: buildMachineClient, registryClient: registryClient, maxConcurrentBuild: maxConcurrentBuild, buildChan: make(chan uint, maxConcurrentBuild), entry: logrus.WithField("service", "Builder"), } go svc.startBuildQueueProcessor() // Start a goroutine to process the queue svc.entry.Info("Builder service initialized and build queue processor started.") return svc } // QueueBuildJob adds a new build job to the queue. // It will first create a BuildJob entry in the database. func (s *Builder) QueueBuildJob(ctx context.Context, req models.BuildRequest) (*models.BuildJob, error) { s.entry.Infof("Received build request for ComponentID: %d, SourceURL: %s", req.ComponentID, req.SourceURL) // Debug: Check if DockerfileContent is present if req.DockerfileContent != "" { lines := strings.Split(req.DockerfileContent, "\n") if len(lines) > 5 { lines = lines[:5] } s.entry.Infof("DockerfileContent received for ComponentID %d (first 5 lines):\n%s", req.ComponentID, strings.Join(lines, "\n")) } else { s.entry.Warnf("DockerfileContent is EMPTY for ComponentID %d", req.ComponentID) } // 1. Validate request if req.ComponentID == 0 || req.SourceURL == "" || req.ImageName == "" { err := fmt.Errorf("invalid build request: ComponentID, SourceURL, and ImageName are required. Got ComponentID: %d, SourceURL: '%s', ImageName: '%s'", req.ComponentID, req.SourceURL, req.ImageName) s.entry.Error(err) return nil, err } // 2. Create BuildJob model imageTag := req.Version if imageTag == "" { imageTag = defaultImageTag } fullImageURI := fmt.Sprintf("%s:%s", req.ImageName, imageTag) if req.RegistryURL != "" { fullImageURI = fmt.Sprintf("%s/%s:%s", req.RegistryURL, req.ImageName, imageTag) } buildArgsJSON := "" if len(req.BuildArgs) > 0 { jsonBytes, err := json.Marshal(req.BuildArgs) if err != nil { s.entry.Errorf("Failed to marshal build args for ComponentID %d: %v", req.ComponentID, err) return nil, fmt.Errorf("failed to marshal build args: %w", err) } buildArgsJSON = string(jsonBytes) } dockerfilePath := req.Dockerfile if dockerfilePath == "" { dockerfilePath = defaultDockerfilePath } buildContext := req.BuildContext if buildContext == "" { buildContext = defaultBuildContext } job := models.BuildJob{ ComponentID: req.ComponentID, RequestID: fmt.Sprintf("build-%d-%s", req.ComponentID, time.Now().Format("20060102150405")), // Unique ID for idempotency SourceURL: req.SourceURL, Status: models.BuildStatusPending, // Corrected: Queued is the initial status set by dbstore.CreateBuildJob ImageName: req.ImageName, ImageTag: imageTag, FullImageURI: fullImageURI, RegistryURL: req.RegistryURL, RegistryUser: req.RegistryUser, // Added RegistryPassword: req.RegistryPassword, // Added BuildContext: buildContext, Dockerfile: dockerfilePath, Source: req.Source, // Added DockerfileContent: req.DockerfileContent, // NEW: Generated Dockerfile content NoCache: req.NoCache, BuildArgs: buildArgsJSON, RequestedAt: time.Now(), } // 3. Save BuildJob to database if err := s.store.CreateBuildJob(ctx, &job); err != nil { s.entry.Errorf("Failed to save build job for ComponentID %d to database: %v", req.ComponentID, err) return nil, fmt.Errorf("failed to save build job: %w", err) } // Debug: Verify the job was saved with DockerfileContent if job.DockerfileContent != "" { s.entry.Infof("Build job ID %d saved with DockerfileContent (length: %d chars)", job.ID, len(job.DockerfileContent)) } else { s.entry.Warnf("Build job ID %d saved with EMPTY DockerfileContent", job.ID) } // 4. Send to buildQueue select { case s.buildChan <- job.ID: // Non-blocking send to channel s.entry.Infof("Build job ID %d for ComponentID %d sent to internal queue.", job.ID, job.ComponentID) default: s.entry.Errorf("Build queue is full. Failed to queue job ID %d for ComponentID %d.", job.ID, job.ComponentID) return &job, fmt.Errorf("build queue is full, cannot process job ID %d at this time", job.ID) } return &job, nil } // startBuildQueueProcessor runs in a goroutine, picking jobs from buildQueue. func (s *Builder) startBuildQueueProcessor() { s.entry.Info("Build queue processor started. Waiting for jobs...") for jobId := range s.buildChan { s.entry.Info("Processing build job from queue.") go s.processJob(context.Background(), jobId) } s.entry.Info("Build queue processor stopped.") } // processJob handles the lifecycle of a single build job. func (s *Builder) processJob(ctx context.Context, jobID uint) { // Implementation for processing a job job, err := s.store.GetBuildJobByID(ctx, jobID) if err != nil { s.entry.Errorf("Failed to retrieve build job ID %d from database: %v", jobID, err) return } s.entry.Infof("Processing build job ID %d for ComponentID %d. Source: %s, BuildContext: %s", job.ID, job.ComponentID, job.Source, job.BuildContext) // Clone repository and resolve build context for docker-compose builds resolvedBuildContext := job.BuildContext var tempRepoDir string if job.Source == "docker-compose" { var err error resolvedBuildContext, err = s.cloneRepositoryAndResolveBuildContext(ctx, job) if err != nil { s.entry.Errorf("Failed to clone repository and resolve build context for job ID %d: %v", job.ID, err) s.updateJobStatus(ctx, job.ID, job.ComponentID, models.BuildStatusFailed, fmt.Sprintf("Failed to resolve build context: %v", err)) s.appendJobLog(ctx, job.ID, fmt.Sprintf("Build failed: %v", err)) return } // Extract temp repo directory for cleanup if strings.Contains(resolvedBuildContext, job.BuildContext) { // The resolved path contains our build context as suffix, extract the temp repo dir tempRepoDir = strings.TrimSuffix(resolvedBuildContext, job.BuildContext) tempRepoDir = strings.TrimSuffix(tempRepoDir, "/") } else if job.BuildContext == "." || job.BuildContext == "" { tempRepoDir = resolvedBuildContext } } // Ensure cleanup of cloned repository after processing if tempRepoDir != "" && tempRepoDir != defaultBuildContext { defer func() { s.entry.Infof("Cleaning up cloned repository directory: %s for job ID %d", tempRepoDir, job.ID) if err := os.RemoveAll(tempRepoDir); err != nil { s.entry.Errorf("Failed to clean up repository directory %s for job ID %d: %v", tempRepoDir, job.ID, err) } else { s.entry.Infof("Successfully cleaned up repository directory: %s for job ID %d", tempRepoDir, job.ID) } }() } s.entry.Infof("Using resolved build context for job ID %d: %s", job.ID, resolvedBuildContext) // Debug: Check if DockerfileContent was retrieved from database if job.DockerfileContent != "" { s.entry.Infof("Job %d retrieved with DockerfileContent (length: %d chars)", job.ID, len(job.DockerfileContent)) lines := strings.Split(job.DockerfileContent, "\n") if len(lines) > 3 { lines = lines[:3] } s.entry.Infof("Job %d DockerfileContent first 3 lines:\n%s", job.ID, strings.Join(lines, "\n")) } else { s.entry.Warnf("Job %d retrieved with EMPTY DockerfileContent", job.ID) } // Update job status to InProgress s.updateJobStatus(ctx, job.ID, job.ComponentID, models.BuildStatusBuilding, "") // Componentend log for job start s.appendJobLog(ctx, job.ID, fmt.Sprintf("Starting build for ComponentID %d from source %s", job.ComponentID, job.SourceURL)) // Parse build arguments buildArgs, err := s.parseBuildArgs(job.BuildArgs) if err != nil { s.entry.Errorf("Failed to parse build args for job ID %d: %v", job.ID, err) s.updateJobStatus(ctx, job.ID, job.ComponentID, models.BuildStatusFailed, fmt.Sprintf("Failed to parse build args: %v", err)) s.appendJobLog(ctx, job.ID, fmt.Sprintf("Build failed: %v", err)) return } // Use resolved build context for the build dockerfilePath := job.Dockerfile if job.Source == "docker-compose" && dockerfilePath != "" && !filepath.IsAbs(dockerfilePath) { // For docker-compose builds, resolve dockerfile path relative to build context dockerfilePath = filepath.Join(resolvedBuildContext, dockerfilePath) } // For registry builds, do build and push in one step for efficiency if job.RegistryURL != "" { s.entry.Infof("Building and pushing image %s to registry %s in one step", job.FullImageURI, job.RegistryURL) buildOutput, err := s.buildAndPushImage(ctx, job, dockerfilePath, resolvedBuildContext, buildArgs) if err != nil { s.entry.Errorf("Build and push failed for job ID %d: %v", job.ID, err) s.updateJobStatus(ctx, job.ID, job.ComponentID, models.BuildStatusFailed, fmt.Sprintf("Build and push failed: %v", err)) s.appendJobLog(ctx, job.ID, fmt.Sprintf("Build and push failed: %v", err)) return } s.entry.Infof("Build and push completed successfully for job ID %d. Output: %s", job.ID, buildOutput) } else { // Local build only buildOutput, err := s.buildMachineClient.BuildImage(ctx, *job, dockerfilePath, resolvedBuildContext, job.ImageName, job.ImageTag, job.NoCache, buildArgs) if err != nil { s.entry.Errorf("Build failed for job ID %d: %v", job.ID, err) s.updateJobStatus(ctx, job.ID, job.ComponentID, models.BuildStatusFailed, fmt.Sprintf("Build failed: %v", err)) s.appendJobLog(ctx, job.ID, fmt.Sprintf("Build failed: %v", err)) return } s.entry.Infof("Build completed successfully for job ID %d. Output: %s", job.ID, buildOutput) } // Debug registry push configuration s.entry.Infof("Registry URL configured: %s", job.RegistryURL) // For registry builds, the build-and-push was already done above // For local builds, no push is needed // Finalize job with success status s.finalizeJob(ctx, job.ID, job.ComponentID, models.BuildStatusSuccess, "") s.appendJobLog(ctx, job.ID, "Build job completed successfully.") s.entry.Infof("Build job ID %d for ComponentID %d completed successfully.", job.ID, job.ComponentID) } // buildAndPushImage builds and pushes an image in one step for docker-compose builds func (s *Builder) buildAndPushImage(ctx context.Context, job *models.BuildJob, dockerfilePath, buildContext string, buildArgs map[string]string) (string, error) { // Create a temporary job with the resolved build context for the build operation tempJob := *job tempJob.BuildContext = buildContext // For DockerfileBuilder, we can call BuildImage with push enabled if dockerfileBuilder, ok := s.buildMachineClient.(*clients.DockerfileBuilder); ok { return s.buildAndPushWithDockerfileBuilder(ctx, dockerfileBuilder, &tempJob, dockerfilePath, buildContext, buildArgs) } // Fallback: build first, then push (original approach) buildOutput, err := s.buildMachineClient.BuildImage(ctx, tempJob, dockerfilePath, buildContext, job.ImageName, job.ImageTag, job.NoCache, buildArgs) if err != nil { return "", fmt.Errorf("build failed: %w", err) } // Push the image if err := s.registryClient.PushImage(ctx, tempJob, job.FullImageURI, job.RegistryURL, job.RegistryUser, job.RegistryPassword); err != nil { return buildOutput, fmt.Errorf("push failed: %w", err) } return buildOutput, nil } // buildAndPushWithDockerfileBuilder builds and pushes using DockerfileBuilder with combined build+push func (s *Builder) buildAndPushWithDockerfileBuilder(ctx context.Context, builder *clients.DockerfileBuilder, job *models.BuildJob, dockerfilePath, buildContext string, buildArgs map[string]string) (string, error) { // We'll create a custom build-and-push operation that uses BuildKit's ability to export directly to registry return builder.BuildImageWithPush(ctx, *job, dockerfilePath, buildContext, job.ImageName, job.ImageTag, job.NoCache, buildArgs, job.FullImageURI, job.RegistryURL, job.RegistryUser, job.RegistryPassword) } // updateJobStatus updates the job's status in the database. func (s *Builder) updateJobStatus(ctx context.Context, jobID uint, componentId uint, status models.BuildStatus, errorMessage string) { if err := s.store.UpdateBuildJobStatus(ctx, jobID, status, errorMessage); err != nil { s.entry.Errorf("Error updating status for build job ID %d to %s: %v", jobID, status, err) } else { s.entry.Infof("Updated status for build job ID %d to %s.", jobID, status) } var componentStatus string switch status { case models.BuildStatusSuccess: componentStatus = "ready" case models.BuildStatusFailed: componentStatus = "failed" default: componentStatus = "in_progress" } if updateErr := s.store.UpdateComponentStatus(ctx, componentId, componentStatus, errorMessage); updateErr != nil { s.entry.Errorf("Error updating component status for job ID %d: %v", jobID, updateErr) } else { s.entry.Infof("Updated component status for job ID %d to %s.", jobID, status) } } // appendJobLog appends a log message to the job's logs in the database. func (s *Builder) appendJobLog(ctx context.Context, jobID uint, message string) { if err := s.store.AppendBuildJobLog(ctx, jobID, message); err != nil { s.entry.Errorf("Error appending log for build job ID %d: %v", jobID, err) s.entry.Infof("[Job %d Log]: %s", jobID, message) } } // finalizeJob sets the final status of the job (Success or Failed) and records FinishedAt. func (s *Builder) finalizeJob(ctx context.Context, jobID uint, componentId uint, status models.BuildStatus, errorMessage string) { if err := s.store.UpdateBuildJobStatus(ctx, jobID, status, errorMessage); err != nil { s.entry.Errorf("Error finalizing build job ID %d with status %s: %v", jobID, status, err) } else { s.entry.Infof("Finalized build job ID %d with status %s.", jobID, status) } var componentStatus string switch status { case models.BuildStatusSuccess: componentStatus = "ready" // Update component with image information if build was successful job, err := s.store.GetBuildJobByID(ctx, jobID) if err != nil { s.entry.Errorf("Error retrieving build job ID %d to update component image info: %v", jobID, err) } else { // Update component with the built image information if err := s.store.UpdateComponentImageInfo(ctx, componentId, job.ImageTag, job.FullImageURI); err != nil { s.entry.Errorf("Error updating component image info for component ID %d after successful build: %v", componentId, err) } else { s.entry.Infof("Successfully updated component ID %d with image tag %s and URI %s", componentId, job.ImageTag, job.FullImageURI) } } case models.BuildStatusFailed: componentStatus = "failed" default: componentStatus = "in_progress" } if updateErr := s.store.UpdateComponentStatus(ctx, componentId, componentStatus, errorMessage); updateErr != nil { s.entry.Errorf("Error updating component status for job ID %d: %v", jobID, updateErr) } else { s.entry.Infof("Updated component status for job ID %d to %s.", jobID, status) } } // parseBuildArgs converts a JSON string of build arguments into a map. func (s *Builder) parseBuildArgs(argsStr string) (map[string]string, error) { if argsStr == "" { return nil, nil } var argsMap map[string]string err := json.Unmarshal([]byte(argsStr), &argsMap) if err != nil { return nil, fmt.Errorf("error unmarshalling build args JSON: %w. JSON string was: %s", err, argsStr) } return argsMap, nil } // cloneRepositoryAndResolveBuildContext clones the repository and resolves the build context // Returns the absolute path to the build context directory func (s *Builder) cloneRepositoryAndResolveBuildContext(ctx context.Context, job *models.BuildJob) (string, error) { // For docker-compose source builds, we need to clone the repository first if job.Source != "docker-compose" { // For non-compose builds, assume build context is already correctly set return job.BuildContext, nil } // Create temporary directory for cloning tempDir, err := os.MkdirTemp("", fmt.Sprintf("byop-build-%d-*", job.ID)) if err != nil { return "", fmt.Errorf("failed to create temp directory: %w", err) } s.entry.Infof("Job %d: Cloning repository %s to %s", job.ID, job.SourceURL, tempDir) // Clone options cloneOptions := &git.CloneOptions{ URL: job.SourceURL, Progress: nil, } // Set branch if specified in version if job.ImageTag != "" && job.ImageTag != "latest" { cloneOptions.ReferenceName = plumbing.NewBranchReferenceName(job.ImageTag) cloneOptions.SingleBranch = true } // Clone the repository _, err = git.PlainClone(tempDir, false, cloneOptions) if err != nil { os.RemoveAll(tempDir) return "", fmt.Errorf("failed to clone repository: %w", err) } // Resolve build context relative to the cloned repository var buildContextPath string if job.BuildContext == "" || job.BuildContext == "." { buildContextPath = tempDir } else { buildContextPath = filepath.Join(tempDir, job.BuildContext) } // Check if build context directory exists if _, err := os.Stat(buildContextPath); os.IsNotExist(err) { os.RemoveAll(tempDir) return "", fmt.Errorf("build context directory does not exist: %s", job.BuildContext) } s.entry.Infof("Job %d: Resolved build context to %s", job.ID, buildContextPath) return buildContextPath, nil }