preview_common.go 35 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026
  1. package services
  2. import (
  3. "archive/tar"
  4. "bytes"
  5. "context"
  6. "crypto/rand"
  7. "encoding/base64"
  8. "encoding/json"
  9. "fmt"
  10. "io"
  11. "os"
  12. "os/exec"
  13. "path/filepath"
  14. "strings"
  15. "time"
  16. "git.linuxforward.com/byop/byop-engine/clients"
  17. "git.linuxforward.com/byop/byop-engine/dbstore"
  18. "git.linuxforward.com/byop/byop-engine/models"
  19. "github.com/sirupsen/logrus"
  20. "github.com/docker/docker/api/types"
  21. "github.com/docker/docker/api/types/container"
  22. "github.com/docker/docker/api/types/filters"
  23. "github.com/docker/docker/api/types/image"
  24. "github.com/docker/docker/api/types/registry"
  25. docker "github.com/docker/docker/client"
  26. )
  27. // PreviewService defines the interface for preview services
  28. type PreviewService interface {
  29. CreatePreview(ctx context.Context, appId int) (*models.Preview, error)
  30. DeletePreview(ctx context.Context, appID int) error
  31. StopPreview(ctx context.Context, previewID int) error
  32. Close(ctx context.Context) // Updated signature to include context
  33. }
  34. // PreviewCommon contains shared functionality for preview services
  35. type PreviewCommon struct {
  36. store *dbstore.SQLiteStore
  37. entry *logrus.Entry
  38. dockerClient *docker.Client
  39. registryClient clients.RegistryClient
  40. registryURL string
  41. registryUser string
  42. registryPass string
  43. }
  44. // NewPreviewCommon creates a new PreviewCommon instance
  45. func NewPreviewCommon(store *dbstore.SQLiteStore, registryClient clients.RegistryClient, registryURL, registryUser, registryPass string) *PreviewCommon {
  46. dockerClient, err := docker.NewClientWithOpts(docker.FromEnv, docker.WithAPIVersionNegotiation())
  47. if err != nil {
  48. logrus.WithError(err).Fatal("Failed to create Docker client")
  49. }
  50. return &PreviewCommon{
  51. store: store,
  52. entry: logrus.WithField("service", "PreviewCommon"),
  53. dockerClient: dockerClient,
  54. registryClient: registryClient,
  55. registryURL: registryURL,
  56. registryUser: registryUser,
  57. registryPass: registryPass,
  58. }
  59. }
  60. // Close cleans up the Docker client connection
  61. func (pc *PreviewCommon) Close() {
  62. // Clean up BYOP preview images
  63. pc.CleanupPreviewImages(context.Background())
  64. // Clean up preview database state
  65. pc.CleanupPreviewState(context.Background())
  66. // Close the Docker client connection
  67. if pc.dockerClient != nil {
  68. if err := pc.dockerClient.Close(); err != nil {
  69. pc.entry.WithError(err).Error("Failed to close Docker client")
  70. } else {
  71. pc.entry.Info("Docker client connection closed")
  72. }
  73. }
  74. }
  75. // GetDockerClient returns the Docker client
  76. func (pc *PreviewCommon) GetDockerClient() *docker.Client {
  77. return pc.dockerClient
  78. }
  79. // GetStore returns the database store
  80. func (pc *PreviewCommon) GetStore() *dbstore.SQLiteStore {
  81. return pc.store
  82. }
  83. // GetLogger returns the logger
  84. func (pc *PreviewCommon) GetLogger() *logrus.Entry {
  85. return pc.entry
  86. }
  87. // GeneratePreviewID generates an 8-character random hex UUID for preview URLs
  88. func (pc *PreviewCommon) GeneratePreviewID() string {
  89. bytes := make([]byte, 4) // 4 bytes = 8 hex chars
  90. if _, err := rand.Read(bytes); err != nil {
  91. // Fallback to timestamp-based ID if crypto/rand fails
  92. return fmt.Sprintf("%08x", time.Now().Unix()%0xFFFFFFFF)
  93. }
  94. // Convert each byte directly to hex to ensure we get truly random looking IDs
  95. return fmt.Sprintf("%02x%02x%02x%02x", bytes[0], bytes[1], bytes[2], bytes[3])
  96. }
  97. // CloneRepository clones a git repository to a target directory
  98. func (pc *PreviewCommon) CloneRepository(ctx context.Context, repoURL, branch, targetDir string) error {
  99. if err := os.MkdirAll(targetDir, 0755); err != nil {
  100. return models.NewErrInternalServer(fmt.Sprintf("failed to create target directory %s", targetDir), err)
  101. }
  102. if branch == "" {
  103. branch = "main"
  104. }
  105. cmd := exec.CommandContext(ctx, "git", "clone", "--depth", "1", "--branch", branch, repoURL, targetDir)
  106. if err := cmd.Run(); err != nil {
  107. // Try with master branch if main fails
  108. if branch == "main" {
  109. cmd = exec.CommandContext(ctx, "git", "clone", "--depth", "1", "--branch", "master", repoURL, targetDir)
  110. if err := cmd.Run(); err != nil {
  111. return models.NewErrInternalServer(fmt.Sprintf("failed to clone repository (tried main and master branches): %s", repoURL), err)
  112. }
  113. } else {
  114. return models.NewErrInternalServer(fmt.Sprintf("failed to clone repository %s on branch %s", repoURL, branch), err)
  115. }
  116. }
  117. return nil
  118. }
  119. // CreateBuildContext creates a tar archive of the build context
  120. func (pc *PreviewCommon) CreateBuildContext(ctx context.Context, contextDir string) (io.ReadCloser, error) {
  121. var buf bytes.Buffer
  122. tw := tar.NewWriter(&buf)
  123. defer tw.Close()
  124. // Common ignore patterns for Git repositories
  125. ignorePatterns := []string{
  126. ".git",
  127. ".gitignore",
  128. "node_modules",
  129. ".next",
  130. "dist",
  131. "build",
  132. "target",
  133. "__pycache__",
  134. "*.pyc",
  135. ".DS_Store",
  136. "Thumbs.db",
  137. "*.log",
  138. "*.tmp",
  139. "*.swp",
  140. ".env",
  141. ".vscode",
  142. ".idea",
  143. "playwright",
  144. "cypress",
  145. "coverage",
  146. "*.test.js",
  147. "*.spec.js",
  148. "*.test.ts",
  149. "*.spec.ts",
  150. "test",
  151. "tests",
  152. "__tests__",
  153. "snapshots",
  154. "*.png",
  155. "*.jpg",
  156. "*.jpeg",
  157. "*.gif",
  158. "*.bmp",
  159. "*.svg",
  160. "*.ico",
  161. "*.zip",
  162. "*.tar.gz",
  163. "*.tar",
  164. "*.gz",
  165. "README.md",
  166. "readme.md",
  167. "CHANGELOG.md",
  168. "LICENSE",
  169. "CONTRIBUTING.md",
  170. "*.md",
  171. "docs",
  172. "documentation",
  173. }
  174. err := filepath.Walk(contextDir, func(file string, fi os.FileInfo, err error) error {
  175. if err != nil {
  176. return err
  177. }
  178. // Get relative path
  179. relPath, err := filepath.Rel(contextDir, file)
  180. if err != nil {
  181. return err
  182. }
  183. // Skip if matches ignore patterns
  184. for _, pattern := range ignorePatterns {
  185. if matched, _ := filepath.Match(pattern, fi.Name()); matched {
  186. if fi.IsDir() {
  187. return filepath.SkipDir
  188. }
  189. return nil
  190. }
  191. if strings.Contains(relPath, pattern) {
  192. if fi.IsDir() {
  193. return filepath.SkipDir
  194. }
  195. return nil
  196. }
  197. }
  198. // Skip very large files (> 100MB)
  199. if !fi.IsDir() && fi.Size() > 100*1024*1024 {
  200. pc.entry.WithField("file", relPath).WithField("size", fi.Size()).Warn("Skipping large file in build context")
  201. return nil
  202. }
  203. // Skip files with very long paths (> 200 chars)
  204. if len(relPath) > 200 {
  205. pc.entry.WithField("file", relPath).WithField("length", len(relPath)).Warn("Skipping file with very long path")
  206. return nil
  207. }
  208. // Create tar header
  209. header, err := tar.FileInfoHeader(fi, fi.Name())
  210. if err != nil {
  211. return err
  212. }
  213. // Update the name to be relative to the context directory
  214. header.Name = filepath.ToSlash(relPath)
  215. // Ensure header name is not too long for tar format
  216. if len(header.Name) > 155 {
  217. pc.entry.WithField("file", header.Name).WithField("length", len(header.Name)).Warn("Skipping file with tar-incompatible long name")
  218. return nil
  219. }
  220. // Write header
  221. if err := tw.WriteHeader(header); err != nil {
  222. return fmt.Errorf("failed to write tar header for %s: %v", relPath, err)
  223. }
  224. // If it's a file, write its content
  225. if !fi.IsDir() {
  226. data, err := os.Open(file)
  227. if err != nil {
  228. return fmt.Errorf("failed to open file %s: %v", relPath, err)
  229. }
  230. defer data.Close()
  231. // Use limited reader to prevent issues with very large files
  232. limitedReader := io.LimitReader(data, 100*1024*1024) // 100MB limit
  233. written, err := io.Copy(tw, limitedReader)
  234. if err != nil {
  235. pc.entry.WithField("file", relPath).WithField("written_bytes", written).Warnf("Failed to copy file to tar, skipping: %v", err)
  236. // Don't return error, just skip this file
  237. return nil
  238. }
  239. }
  240. return nil
  241. })
  242. if err != nil {
  243. return nil, err
  244. }
  245. return io.NopCloser(&buf), nil
  246. }
  247. // createDockerIgnore creates a .dockerignore file in the specified directory
  248. func (pc *PreviewCommon) createDockerIgnore(ctx context.Context, contextDir string) {
  249. dockerignoreContent := `# Auto-generated by BYOP Engine
  250. .git
  251. .gitignore
  252. node_modules
  253. .next
  254. dist
  255. build
  256. target
  257. __pycache__
  258. *.pyc
  259. .DS_Store
  260. Thumbs.db
  261. *.log
  262. *.tmp
  263. *.swp
  264. .env
  265. .vscode
  266. .idea
  267. playwright
  268. cypress
  269. coverage
  270. test
  271. tests
  272. __tests__
  273. snapshots
  274. *.test.js
  275. *.spec.js
  276. *.test.ts
  277. *.spec.ts
  278. *.png
  279. *.jpg
  280. *.jpeg
  281. *.gif
  282. *.bmp
  283. *.svg
  284. *.ico
  285. *.zip
  286. *.tar.gz
  287. *.tar
  288. *.gz
  289. README.md
  290. readme.md
  291. CHANGELOG.md
  292. LICENSE
  293. CONTRIBUTING.md
  294. *.md
  295. docs
  296. documentation
  297. `
  298. dockerignorePath := filepath.Join(contextDir, ".dockerignore")
  299. if err := os.WriteFile(dockerignorePath, []byte(dockerignoreContent), 0644); err != nil {
  300. pc.entry.WithField("path", dockerignorePath).Warnf("Failed to create .dockerignore file: %v", err)
  301. } else {
  302. pc.entry.WithField("path", dockerignorePath).Debug("Created .dockerignore file")
  303. }
  304. }
  305. // validateAndFixDockerfile checks for unsupported Dockerfile syntax and fixes common issues
  306. func (pc *PreviewCommon) validateAndFixDockerfile(ctx context.Context, contextDir string) error {
  307. dockerfilePath := filepath.Join(contextDir, "Dockerfile")
  308. // Check if Dockerfile exists
  309. if _, err := os.Stat(dockerfilePath); os.IsNotExist(err) {
  310. return fmt.Errorf("dockerfile not found in repository")
  311. }
  312. // Read the Dockerfile
  313. content, err := os.ReadFile(dockerfilePath)
  314. if err != nil {
  315. return fmt.Errorf("failed to read Dockerfile: %v", err)
  316. }
  317. originalContent := string(content)
  318. modifiedContent := originalContent
  319. modified := false
  320. // Fix common issues
  321. lines := strings.Split(originalContent, "\n")
  322. var fixedLines []string
  323. for i, line := range lines {
  324. trimmedLine := strings.TrimSpace(line)
  325. // Check for unsupported --exclude flag in COPY or ADD commands
  326. if strings.HasPrefix(trimmedLine, "COPY") || strings.HasPrefix(trimmedLine, "ADD") {
  327. if strings.Contains(trimmedLine, "--exclude") {
  328. pc.entry.WithField("line", i+1).Warn("Found unsupported --exclude flag in Dockerfile, removing it")
  329. // Remove --exclude flag and its arguments
  330. parts := strings.Fields(trimmedLine)
  331. var cleanedParts []string
  332. skipNext := false
  333. for _, part := range parts {
  334. if skipNext {
  335. skipNext = false
  336. continue
  337. }
  338. if strings.HasPrefix(part, "--exclude") {
  339. if strings.Contains(part, "=") {
  340. // --exclude=pattern format
  341. continue
  342. } else {
  343. // --exclude pattern format
  344. skipNext = true
  345. continue
  346. }
  347. }
  348. cleanedParts = append(cleanedParts, part)
  349. }
  350. fixedLine := strings.Join(cleanedParts, " ")
  351. fixedLines = append(fixedLines, fixedLine)
  352. modified = true
  353. pc.entry.WithField("original", trimmedLine).WithField("fixed", fixedLine).Info("Fixed Dockerfile line")
  354. } else {
  355. fixedLines = append(fixedLines, line)
  356. }
  357. } else {
  358. fixedLines = append(fixedLines, line)
  359. }
  360. }
  361. // Write back the fixed Dockerfile if modified
  362. if modified {
  363. modifiedContent = strings.Join(fixedLines, "\n")
  364. if err := os.WriteFile(dockerfilePath, []byte(modifiedContent), 0644); err != nil {
  365. return fmt.Errorf("failed to write fixed Dockerfile: %v", err)
  366. }
  367. pc.entry.WithField("path", dockerfilePath).Info("Fixed Dockerfile syntax issues")
  368. }
  369. return nil
  370. }
  371. // isDockerfilePresent checks if a Dockerfile exists in the repository directory
  372. func (pc *PreviewCommon) isDockerfilePresent(tempDir string) (bool, error) {
  373. // Check for common Dockerfile names
  374. dockerfileNames := []string{"Dockerfile", "dockerfile", "Dockerfile.prod", "Dockerfile.production"}
  375. for _, name := range dockerfileNames {
  376. dockerfilePath := filepath.Join(tempDir, name)
  377. if _, err := os.Stat(dockerfilePath); err == nil {
  378. pc.entry.WithField("dockerfile_path", dockerfilePath).Debug("Found Dockerfile")
  379. return true, nil
  380. }
  381. }
  382. pc.entry.WithField("temp_dir", tempDir).Debug("No Dockerfile found")
  383. return false, nil
  384. }
  385. // BuildComponentImages builds Docker images for components
  386. // It first checks for pre-built images in the registry before rebuilding from source
  387. func (pc *PreviewCommon) BuildComponentImages(ctx context.Context, components []models.Component) ([]string, string, error) {
  388. var imageNames []string
  389. var allLogs strings.Builder
  390. for _, component := range components {
  391. pc.entry.WithField("component_id", component.ID).WithField("status", component.Status).Info("Processing component for preview")
  392. // Generate local image name for preview
  393. imageName := fmt.Sprintf("byop-preview-%s:%d", component.Name, component.ID)
  394. // Check if component has pre-built image information
  395. if component.CurrentImageURI != "" && component.CurrentImageTag != "" {
  396. pc.entry.WithField("component_id", component.ID).WithField("image_uri", component.CurrentImageURI).Info("Component has pre-built image, checking registry")
  397. allLogs.WriteString(fmt.Sprintf("Component %d has pre-built image %s, checking availability\n", component.ID, component.CurrentImageURI))
  398. // Check if the pre-built image exists in the registry
  399. if pc.registryClient != nil && pc.registryURL != "" {
  400. exists, err := pc.registryClient.CheckImageExists(ctx, component.CurrentImageURI, pc.registryURL, pc.registryUser, pc.registryPass)
  401. if err != nil {
  402. pc.entry.WithField("component_id", component.ID).WithError(err).Warn("Failed to check if pre-built image exists, falling back to rebuild")
  403. allLogs.WriteString(fmt.Sprintf("Failed to check registry image for component %d: %v, rebuilding from source\n", component.ID, err))
  404. } else if exists {
  405. // Pull the pre-built image from registry to local Docker
  406. if err := pc.pullPreBuiltImage(ctx, component.CurrentImageURI, imageName); err != nil {
  407. pc.entry.WithField("component_id", component.ID).WithError(err).Warn("Failed to pull pre-built image, falling back to rebuild")
  408. allLogs.WriteString(fmt.Sprintf("Failed to pull pre-built image for component %d: %v, rebuilding from source\n", component.ID, err))
  409. } else {
  410. pc.entry.WithField("component_id", component.ID).WithField("image_name", imageName).Info("Successfully used pre-built image")
  411. allLogs.WriteString(fmt.Sprintf("Successfully pulled and tagged pre-built image for component %d as %s\n", component.ID, imageName))
  412. imageNames = append(imageNames, imageName)
  413. continue // Skip to next component
  414. }
  415. } else {
  416. pc.entry.WithField("component_id", component.ID).Info("Pre-built image not found in registry, rebuilding from source")
  417. allLogs.WriteString(fmt.Sprintf("Pre-built image for component %d not found in registry, rebuilding from source\n", component.ID))
  418. }
  419. } else {
  420. pc.entry.WithField("component_id", component.ID).Warn("Registry client not configured, cannot check pre-built images")
  421. allLogs.WriteString(fmt.Sprintf("Registry not configured for component %d, rebuilding from source\n", component.ID))
  422. }
  423. } else {
  424. pc.entry.WithField("component_id", component.ID).Info("Component has no pre-built image information, building from source")
  425. allLogs.WriteString(fmt.Sprintf("Component %d has no pre-built image, building from source\n", component.ID))
  426. }
  427. // Fallback: Build from source code
  428. pc.entry.WithField("component_id", component.ID).Info("Building Docker image from source")
  429. allLogs.WriteString(fmt.Sprintf("Building component %d from source\n", component.ID))
  430. // Create temp directory for this component
  431. tempDir := filepath.Join(os.TempDir(), fmt.Sprintf("byop-preview-%d-%d", component.ID, time.Now().Unix()))
  432. defer os.RemoveAll(tempDir)
  433. // Clone repository
  434. if err := pc.CloneRepository(ctx, component.Repository, component.Branch, tempDir); err != nil {
  435. allLogs.WriteString(fmt.Sprintf("Failed to clone %s: %v\n", component.Repository, err))
  436. return nil, allLogs.String(), err
  437. }
  438. // Special handling for components with existing Dockerfiles (status "valid")
  439. if component.Status == "valid" {
  440. pc.entry.WithField("component_id", component.ID).Info("Component has existing Dockerfile, building directly")
  441. allLogs.WriteString(fmt.Sprintf("Component %d has existing Dockerfile, building directly\n", component.ID))
  442. // For components with existing Dockerfiles, just use the Dockerfile as-is
  443. // No need to validate/fix or create .dockerignore since they should work as-is
  444. } else {
  445. // For components without existing Dockerfiles (generated via LLB), apply fixes
  446. pc.entry.WithField("component_id", component.ID).Info("Component using generated Dockerfile, applying fixes")
  447. // Create .dockerignore file to exclude unnecessary files
  448. pc.createDockerIgnore(ctx, tempDir)
  449. // Check and fix Dockerfile if needed
  450. if err := pc.validateAndFixDockerfile(ctx, tempDir); err != nil {
  451. allLogs.WriteString(fmt.Sprintf("Failed to validate Dockerfile for component %d: %v\n", component.ID, err))
  452. return nil, allLogs.String(), err
  453. }
  454. }
  455. // Create a tar archive of the build context
  456. pc.entry.WithField("component_id", component.ID).Info("Creating build context tar archive")
  457. tarReader, err := pc.CreateBuildContext(ctx, tempDir)
  458. if err != nil {
  459. errMsg := fmt.Sprintf("Failed to create build context for %s: %v", imageName, err)
  460. pc.entry.WithField("component_id", component.ID).Error(errMsg)
  461. allLogs.WriteString(errMsg + "\n")
  462. return nil, allLogs.String(), err
  463. }
  464. defer tarReader.Close()
  465. pc.entry.WithField("component_id", component.ID).WithField("image_name", imageName).Info("Starting Docker image build")
  466. buildResponse, err := pc.dockerClient.ImageBuild(ctx, tarReader, types.ImageBuildOptions{
  467. Tags: []string{imageName},
  468. Dockerfile: "Dockerfile",
  469. Remove: true,
  470. ForceRemove: true,
  471. })
  472. if err != nil {
  473. errMsg := fmt.Sprintf("Failed to start build for %s: %v", imageName, err)
  474. pc.entry.WithField("component_id", component.ID).Error(errMsg)
  475. allLogs.WriteString(errMsg + "\n")
  476. return nil, allLogs.String(), err
  477. }
  478. defer buildResponse.Body.Close()
  479. // Read and parse build output properly
  480. buildOutput, err := io.ReadAll(buildResponse.Body)
  481. if err != nil {
  482. allLogs.WriteString(fmt.Sprintf("Failed to read build output for %s: %v\n", imageName, err))
  483. return nil, allLogs.String(), err
  484. }
  485. buildOutputStr := string(buildOutput)
  486. allLogs.WriteString(fmt.Sprintf("Building %s:\n%s\n", imageName, buildOutputStr))
  487. // Check for Docker build errors in JSON output
  488. buildSuccess := false
  489. buildErrorFound := false
  490. // Parse each line of JSON output
  491. lines := strings.Split(buildOutputStr, "\n")
  492. for _, line := range lines {
  493. line = strings.TrimSpace(line)
  494. if line == "" {
  495. continue
  496. }
  497. // Look for success indicators
  498. if strings.Contains(line, `"stream":"Successfully built`) ||
  499. strings.Contains(line, `"stream":"Successfully tagged`) {
  500. buildSuccess = true
  501. }
  502. // Look for error indicators
  503. if strings.Contains(line, `"error"`) ||
  504. strings.Contains(line, `"errorDetail"`) ||
  505. strings.Contains(line, `"stream":"ERROR`) ||
  506. strings.Contains(line, `"stream":"The command"`) && strings.Contains(line, "returned a non-zero code") {
  507. buildErrorFound = true
  508. allLogs.WriteString(fmt.Sprintf("Build error detected in line: %s\n", line))
  509. }
  510. }
  511. if buildErrorFound {
  512. allLogs.WriteString(fmt.Sprintf("Build failed for %s: errors found in build output\n", imageName))
  513. return nil, allLogs.String(), fmt.Errorf("docker build failed for %s: check build logs", imageName)
  514. }
  515. if !buildSuccess {
  516. allLogs.WriteString(fmt.Sprintf("Build failed for %s: no success indicators found in build output\n", imageName))
  517. return nil, allLogs.String(), fmt.Errorf("docker build failed for %s: build did not complete successfully", imageName)
  518. }
  519. // Verify the image exists and is properly tagged
  520. _, err = pc.dockerClient.ImageInspect(ctx, imageName)
  521. if err != nil {
  522. allLogs.WriteString(fmt.Sprintf("Build verification failed for %s: image not found after build - %v\n", imageName, err))
  523. return nil, allLogs.String(), fmt.Errorf("failed to build image %s: image not found after build", imageName)
  524. }
  525. imageNames = append(imageNames, imageName)
  526. pc.entry.WithField("component_id", component.ID).WithField("image_name", imageName).Info("Successfully built Docker image")
  527. }
  528. return imageNames, allLogs.String(), nil
  529. }
  530. // pullPreBuiltImage pulls a pre-built image from the registry and tags it for local use
  531. func (pc *PreviewCommon) pullPreBuiltImage(ctx context.Context, registryImageURI, localImageName string) error {
  532. pc.entry.WithField("registry_image", registryImageURI).WithField("local_image", localImageName).Info("Pulling pre-built image from registry")
  533. // Pull the image from registry
  534. pullOptions := image.PullOptions{}
  535. // Add authentication if registry credentials are configured
  536. if pc.registryUser != "" && pc.registryPass != "" {
  537. authConfig := registry.AuthConfig{
  538. Username: pc.registryUser,
  539. Password: pc.registryPass,
  540. }
  541. encodedJSON, err := json.Marshal(authConfig)
  542. if err != nil {
  543. return fmt.Errorf("failed to encode registry auth: %w", err)
  544. }
  545. pullOptions.RegistryAuth = base64.URLEncoding.EncodeToString(encodedJSON)
  546. }
  547. reader, err := pc.dockerClient.ImagePull(ctx, registryImageURI, pullOptions)
  548. if err != nil {
  549. return fmt.Errorf("failed to pull image %s: %w", registryImageURI, err)
  550. }
  551. defer reader.Close()
  552. // Read the pull output (similar to build output)
  553. pullOutput, err := io.ReadAll(reader)
  554. if err != nil {
  555. return fmt.Errorf("failed to read pull output: %w", err)
  556. }
  557. pc.entry.WithField("pull_output", string(pullOutput)).Debug("Image pull completed")
  558. // Tag the pulled image with the local preview tag
  559. err = pc.dockerClient.ImageTag(ctx, registryImageURI, localImageName)
  560. if err != nil {
  561. return fmt.Errorf("failed to tag image %s as %s: %w", registryImageURI, localImageName, err)
  562. }
  563. // Verify the image is now available locally
  564. _, err = pc.dockerClient.ImageInspect(ctx, localImageName)
  565. if err != nil {
  566. return fmt.Errorf("failed to verify locally tagged image %s: %w", localImageName, err)
  567. }
  568. pc.entry.WithField("local_image", localImageName).Info("Successfully pulled and tagged pre-built image")
  569. return nil
  570. }
  571. // GetAppComponents retrieves components for an app
  572. func (pc *PreviewCommon) GetAppComponents(ctx context.Context, app *models.App) ([]models.Component, error) {
  573. var components []models.Component
  574. for _, componentID := range app.Components {
  575. component, err := pc.store.GetComponentByID(ctx, componentID)
  576. if err != nil {
  577. return nil, err
  578. }
  579. if component == nil {
  580. return nil, models.NewErrNotFound(fmt.Sprintf("Component with ID %d not found while fetching app components", componentID), nil)
  581. }
  582. components = append(components, *component)
  583. }
  584. return components, nil
  585. }
  586. // CleanupPreviewImages cleans up BYOP preview Docker images
  587. func (pc *PreviewCommon) CleanupPreviewImages(ctx context.Context) {
  588. pc.entry.Info("Cleaning up BYOP preview images...")
  589. images, err := pc.dockerClient.ImageList(ctx, image.ListOptions{All: true})
  590. if err != nil {
  591. pc.entry.WithError(err).Error("Failed to list images for cleanup")
  592. return
  593. }
  594. removedCount := 0
  595. for _, img := range images {
  596. // Check if image name contains "byop-preview"
  597. isPreviewImage := false
  598. for _, tag := range img.RepoTags {
  599. if strings.Contains(tag, "byop-preview") {
  600. isPreviewImage = true
  601. break
  602. }
  603. }
  604. if !isPreviewImage {
  605. continue
  606. }
  607. // Remove the image
  608. if _, err := pc.dockerClient.ImageRemove(ctx, img.ID, image.RemoveOptions{
  609. Force: true,
  610. PruneChildren: true,
  611. }); err != nil {
  612. pc.entry.WithError(err).WithField("image_id", img.ID).Warn("Failed to remove preview image")
  613. } else {
  614. removedCount++
  615. }
  616. }
  617. if removedCount > 0 {
  618. pc.entry.WithField("removed_images", removedCount).Info("Cleaned up BYOP preview images")
  619. }
  620. }
  621. // CleanupByAppID cleans up all BYOP preview containers and images for a specific app ID
  622. func (pc *PreviewCommon) CleanupByAppID(ctx context.Context, appID int) {
  623. pc.entry.WithField("app_id", appID).Info("Cleaning up BYOP preview containers...")
  624. // List all containers
  625. containers, err := pc.dockerClient.ContainerList(ctx, container.ListOptions{All: true})
  626. if err != nil {
  627. pc.entry.WithError(err).Error("Failed to list containers for cleanup")
  628. return
  629. }
  630. for _, ctn := range containers {
  631. isPreviewContainer := false
  632. containerName := ""
  633. // Check if the container is a BYOP preview container
  634. for key, value := range ctn.Labels {
  635. if key == "byop.preview" && value == "true" {
  636. isPreviewContainer = true
  637. if len(ctn.Names) > 0 {
  638. containerName = ctn.Names[0]
  639. }
  640. break
  641. }
  642. }
  643. if !isPreviewContainer {
  644. continue
  645. }
  646. if ctn.Labels["byop.app.id"] != fmt.Sprintf("%d", appID) {
  647. continue // Only clean up containers for the specified app ID
  648. }
  649. pc.entry.WithField("container_id", ctn.ID).WithField("container_name", containerName).Info("Removing BYOP preview container")
  650. // Remove the container
  651. if err := pc.dockerClient.ContainerRemove(ctx, ctn.ID, container.RemoveOptions{
  652. Force: true,
  653. }); err != nil {
  654. pc.entry.WithError(err).WithField("container_id", ctn.ID).Warn("Failed to remove preview container")
  655. } else {
  656. pc.entry.WithField("container_id", ctn.ID).Info("Successfully removed preview container")
  657. }
  658. }
  659. }
  660. // CleanupAllPreviewContainers cleans up all BYOP preview containers
  661. func (pc *PreviewCommon) CleanupAllPreviewContainers(ctx context.Context) {
  662. pc.entry.Info("Cleaning up all BYOP preview containers...")
  663. // Get all containers with filters for BYOP preview containers
  664. containers, err := pc.dockerClient.ContainerList(ctx, container.ListOptions{
  665. All: true, // Include stopped containers too
  666. Filters: filters.NewArgs(
  667. filters.Arg("label", "byop.preview=true"),
  668. ),
  669. })
  670. if err != nil {
  671. pc.entry.WithError(err).Error("Failed to list BYOP preview containers")
  672. // Fallback to name-based filtering if labels don't work
  673. pc.cleanupByName(ctx)
  674. return
  675. }
  676. if len(containers) == 0 {
  677. pc.entry.Info("No BYOP preview containers found to cleanup")
  678. } else {
  679. pc.entry.WithField("container_count", len(containers)).Info("Found BYOP preview containers to cleanup")
  680. }
  681. // Remove BYOP preview containers
  682. for _, ctn := range containers {
  683. containerName := "unknown"
  684. if len(ctn.Names) > 0 {
  685. containerName = strings.TrimPrefix(ctn.Names[0], "/")
  686. }
  687. pc.entry.WithField("container_id", ctn.ID).WithField("container_name", containerName).Info("Removing BYOP preview container")
  688. // Stop container first if it's running
  689. if ctn.State == "running" {
  690. if err := pc.dockerClient.ContainerStop(ctx, ctn.ID, container.StopOptions{}); err != nil {
  691. pc.entry.WithError(err).WithField("container_id", ctn.ID).Warn("Failed to stop container, will force remove")
  692. }
  693. }
  694. // Remove container
  695. if err := pc.dockerClient.ContainerRemove(ctx, ctn.ID, container.RemoveOptions{
  696. Force: true,
  697. RemoveVolumes: true,
  698. }); err != nil {
  699. pc.entry.WithError(err).WithField("container_id", ctn.ID).Error("Failed to remove BYOP preview container")
  700. } else {
  701. pc.entry.WithField("container_id", ctn.ID).WithField("container_name", containerName).Info("Successfully removed BYOP preview container")
  702. }
  703. }
  704. }
  705. // Fallback method to cleanup containers by name pattern
  706. func (pc *PreviewCommon) cleanupByName(ctx context.Context) {
  707. pc.entry.Info("Using fallback name-based container cleanup")
  708. containers, err := pc.dockerClient.ContainerList(ctx, container.ListOptions{All: true})
  709. if err != nil {
  710. pc.entry.WithError(err).Error("Failed to list containers for name-based cleanup")
  711. return
  712. }
  713. for _, ctn := range containers {
  714. // Check if any container name contains "byop-preview"
  715. isPreviewContainer := false
  716. containerName := "unknown"
  717. for _, name := range ctn.Names {
  718. cleanName := strings.TrimPrefix(name, "/")
  719. if strings.Contains(cleanName, "byop-preview") || strings.Contains(cleanName, "preview") {
  720. isPreviewContainer = true
  721. containerName = cleanName
  722. break
  723. }
  724. }
  725. if !isPreviewContainer {
  726. continue
  727. }
  728. pc.entry.WithField("container_id", ctn.ID).WithField("container_name", containerName).Info("Removing BYOP preview container (name-based)")
  729. // Stop and remove
  730. if ctn.State == "running" {
  731. pc.dockerClient.ContainerStop(ctx, ctn.ID, container.StopOptions{})
  732. }
  733. if err := pc.dockerClient.ContainerRemove(ctx, ctn.ID, container.RemoveOptions{
  734. Force: true,
  735. RemoveVolumes: true,
  736. }); err != nil {
  737. pc.entry.WithError(err).WithField("container_id", ctn.ID).Error("Failed to remove container")
  738. }
  739. }
  740. }
  741. // CleanupPreviewState cleans up preview database state - mark all running previews as stopped
  742. func (pc *PreviewCommon) CleanupPreviewState(ctx context.Context) {
  743. pc.entry.Info("Cleaning up preview database state...")
  744. // Get all active previews (building, deploying, running)
  745. activeStatuses := []string{"building", "deploying", "running"}
  746. for _, status := range activeStatuses {
  747. previews, err := pc.store.GetPreviewsByStatus(ctx, status)
  748. if err != nil {
  749. pc.entry.WithError(err).WithField("status", status).Error("Failed to get previews by status")
  750. continue
  751. }
  752. for _, preview := range previews {
  753. pc.entry.WithField("preview_id", preview.ID).WithField("app_id", preview.AppID).WithField("old_status", preview.Status).Info("Marking preview as stopped due to server shutdown")
  754. // Update preview status to stopped
  755. if err := pc.store.UpdatePreviewStatus(ctx, preview.ID, "stopped", "Server shutdown - containers may have been stopped"); err != nil {
  756. pc.entry.WithError(err).WithField("preview_id", preview.ID).Error("Failed to update preview status to stopped")
  757. }
  758. // Also update the associated app status back to "ready" if it was in a preview state
  759. if app, err := pc.store.GetAppByID(ctx, preview.AppID); err == nil && app != nil {
  760. if app.Status == "building" || app.Status == "deploying" {
  761. if err := pc.store.UpdateAppStatus(ctx, app.ID, "ready", ""); err != nil {
  762. pc.entry.WithError(err).WithField("app_id", app.ID).Error("Failed to reset app status to ready")
  763. } else {
  764. pc.entry.WithField("app_id", app.ID).Info("Reset app status to ready after preview cleanup")
  765. }
  766. }
  767. }
  768. }
  769. if len(previews) > 0 {
  770. pc.entry.WithField("count", len(previews)).WithField("status", status).Info("Updated preview statuses to stopped")
  771. }
  772. }
  773. pc.entry.Info("Preview database state cleanup completed")
  774. }
  775. // GetPreviewImageNames reconstructs the Docker image names used for a preview
  776. func (pc *PreviewCommon) GetPreviewImageNames(appID int) ([]string, error) {
  777. // Get app details
  778. app, err := pc.store.GetAppByID(context.Background(), appID)
  779. if err != nil {
  780. return nil, fmt.Errorf("failed to get app by ID %d: %v", appID, err)
  781. }
  782. // Get all components for the app
  783. components, err := pc.GetAppComponents(context.Background(), app)
  784. if err != nil {
  785. return nil, fmt.Errorf("failed to get app components: %v", err)
  786. }
  787. // Reconstruct image names using the same format as BuildComponentImages
  788. var imageNames []string
  789. for _, component := range components {
  790. imageName := fmt.Sprintf("byop-preview-%s:%d", component.Name, component.ID)
  791. imageNames = append(imageNames, imageName)
  792. }
  793. return imageNames, nil
  794. }
  795. // CleanupPreviewImagesForApp cleans up Docker images for a specific app (works for both local and remote)
  796. func (pc *PreviewCommon) CleanupPreviewImagesForApp(ctx context.Context, appID int, isRemote bool, ipAddress string) error {
  797. imageNames, err := pc.GetPreviewImageNames(appID)
  798. if err != nil {
  799. pc.entry.WithField("app_id", appID).WithError(err).Warn("Failed to get preview image names for cleanup")
  800. return err
  801. }
  802. if isRemote && ipAddress != "" && ipAddress != "127.0.0.1" {
  803. return pc.cleanupRemoteDockerImages(ctx, ipAddress, imageNames)
  804. } else {
  805. return pc.cleanupLocalDockerImages(ctx, imageNames)
  806. }
  807. }
  808. // cleanupLocalDockerImages removes specific Docker images locally
  809. func (pc *PreviewCommon) cleanupLocalDockerImages(ctx context.Context, imageNames []string) error {
  810. pc.entry.WithField("image_count", len(imageNames)).Info("Cleaning up specific Docker images locally")
  811. for _, imageName := range imageNames {
  812. // Remove the image locally using Docker client
  813. if _, err := pc.dockerClient.ImageRemove(ctx, imageName, image.RemoveOptions{
  814. Force: true,
  815. PruneChildren: true,
  816. }); err != nil {
  817. // Log warning but don't fail the cleanup - image might already be removed or in use
  818. pc.entry.WithField("image_name", imageName).WithError(err).Warn("Failed to remove Docker image locally (this may be normal)")
  819. } else {
  820. pc.entry.WithField("image_name", imageName).Info("Successfully removed Docker image locally")
  821. }
  822. }
  823. return nil
  824. }
  825. // cleanupRemoteDockerImages removes Docker images from a VPS via SSH
  826. func (pc *PreviewCommon) cleanupRemoteDockerImages(ctx context.Context, ipAddress string, imageNames []string) error {
  827. pc.entry.WithField("ip_address", ipAddress).WithField("image_count", len(imageNames)).Info("Cleaning up Docker images on VPS")
  828. for _, imageName := range imageNames {
  829. // Remove the image
  830. rmImageCmd := fmt.Sprintf("docker rmi %s --force", imageName)
  831. pc.entry.WithField("image_name", imageName).WithField("ip_address", ipAddress).Info("Removing Docker image")
  832. if err := pc.executeSSHCommand(ctx, ipAddress, rmImageCmd); err != nil {
  833. // Log warning but don't fail the cleanup - image might already be removed or in use
  834. pc.entry.WithField("image_name", imageName).WithField("ip_address", ipAddress).WithError(err).Warn("Failed to remove Docker image (this may be normal)")
  835. } else {
  836. pc.entry.WithField("image_name", imageName).WithField("ip_address", ipAddress).Info("Successfully removed Docker image")
  837. }
  838. // Also remove the tar file if it exists
  839. tarFileName := strings.ReplaceAll(imageName, ":", "_")
  840. rmTarCmd := fmt.Sprintf("rm -f /tmp/%s.tar", tarFileName)
  841. pc.executeSSHCommand(ctx, ipAddress, rmTarCmd) // Ignore errors for tar cleanup
  842. }
  843. // Clean up any dangling images
  844. pc.entry.WithField("ip_address", ipAddress).Info("Cleaning up dangling Docker images")
  845. danglingCmd := "docker image prune -f"
  846. if err := pc.executeSSHCommand(ctx, ipAddress, danglingCmd); err != nil {
  847. pc.entry.WithField("ip_address", ipAddress).WithError(err).Warn("Failed to clean dangling images")
  848. }
  849. return nil
  850. }
  851. // executeSSHCommand executes a command on a remote VPS via SSH
  852. func (pc *PreviewCommon) executeSSHCommand(ctx context.Context, ipAddress, command string) error {
  853. pc.entry.WithField("ip_address", ipAddress).WithField("command", command).Debug("Executing SSH command")
  854. cmd := exec.CommandContext(ctx, "ssh", "-o", "StrictHostKeyChecking=no", ipAddress, command)
  855. output, err := cmd.CombinedOutput()
  856. if err != nil {
  857. pc.entry.WithField("ip_address", ipAddress).WithField("command", command).WithField("output", string(output)).WithError(err).Error("SSH command failed")
  858. return models.NewErrInternalServer(fmt.Sprintf("SSH command failed on %s: %s. Output: %s", ipAddress, command, string(output)), err)
  859. }
  860. if len(output) > 0 {
  861. pc.entry.WithField("ip_address", ipAddress).WithField("command", command).WithField("output", string(output)).Debug("SSH command output")
  862. }
  863. return nil
  864. }
  865. // Database helper methods
  866. func (pc *PreviewCommon) UpdatePreviewStatus(ctx context.Context, previewID int, status, errorMsg string) {
  867. if err := pc.store.UpdatePreviewStatus(ctx, previewID, status, errorMsg); err != nil {
  868. pc.entry.WithField("preview_id", previewID).Errorf("Failed to update preview status: %v", err)
  869. }
  870. }
  871. func (pc *PreviewCommon) UpdatePreviewBuildLogs(ctx context.Context, previewID int, logs string) {
  872. if err := pc.store.UpdatePreviewBuildLogs(ctx, previewID, logs); err != nil {
  873. pc.entry.WithField("preview_id", previewID).Errorf("Failed to update preview build logs: %v", err)
  874. }
  875. }
  876. func (pc *PreviewCommon) UpdatePreviewDeployLogs(ctx context.Context, previewID int, logs string) {
  877. if err := pc.store.UpdatePreviewDeployLogs(ctx, previewID, logs); err != nil {
  878. pc.entry.WithField("preview_id", previewID).Errorf("Failed to update preview deploy logs: %v", err)
  879. }
  880. }