From b40f5b788c16b4071f569173bc5748410313e680 Mon Sep 17 00:00:00 2001 From: Michel Roegl-Brunner Date: Fri, 24 Oct 2025 21:58:35 +0200 Subject: [PATCH] Fix script downloader placeholder files issue - Delete stub scriptDownloader.js that contained placeholder implementation - Implement real JavaScript script downloader with GitHub fetch functionality - Fix incremental JSON sync to only process newly synced files - Add proper error handling and file structure management - Support all script types (ct/, tools/, vm/, vw/) with directory preservation - Download install scripts for CT scripts - Re-enable auto-sync service to use real implementation Scripts now download real content from GitHub instead of placeholders. --- src/server/api/routers/scripts.ts | 6 +- src/server/lib/scripts.ts | 25 +- src/server/services/autoSyncService.js | 161 ++++++++++--- src/server/services/githubJsonService.js | 278 +---------------------- src/server/services/githubJsonService.ts | 92 ++++++-- src/server/services/localScripts.js | 6 + src/server/services/scriptDownloader.js | 183 +++------------ 7 files changed, 260 insertions(+), 491 deletions(-) create mode 100644 src/server/services/localScripts.js diff --git a/src/server/api/routers/scripts.ts b/src/server/api/routers/scripts.ts index 61947d4..452989c 100644 --- a/src/server/api/routers/scripts.ts +++ b/src/server/api/routers/scripts.ts @@ -1,9 +1,9 @@ import { z } from "zod"; import { createTRPCRouter, publicProcedure } from "~/server/api/trpc"; import { scriptManager } from "~/server/lib/scripts"; -import { githubJsonService } from "~/server/services/githubJsonService.ts"; -import { localScriptsService } from "~/server/services/localScripts.ts"; -import { scriptDownloaderService } from "~/server/services/scriptDownloader.ts"; +import { githubJsonService } from "~/server/services/githubJsonService"; +import { localScriptsService } from "~/server/services/localScripts"; +import { scriptDownloaderService } from "~/server/services/scriptDownloader"; import { AutoSyncService } from "~/server/services/autoSyncService"; import type { ScriptCard } from "~/types/script"; diff --git a/src/server/lib/scripts.ts b/src/server/lib/scripts.ts index 33038f1..6cc408c 100644 --- a/src/server/lib/scripts.ts +++ b/src/server/lib/scripts.ts @@ -25,6 +25,25 @@ export class ScriptManager { // Initialize lazily to avoid accessing env vars during module load } + /** + * Safely handle file modification time, providing fallback for invalid dates + * @param mtime - The file modification time from fs.stat + * @returns Date - Valid date or current date as fallback + */ + private safeMtime(mtime: Date): Date { + try { + // Check if the date is valid + if (!mtime || isNaN(mtime.getTime())) { + console.warn('Invalid mtime detected, using current time as fallback'); + return new Date(); + } + return mtime; + } catch (error) { + console.warn('Error processing mtime:', error); + return new Date(); + } + } + private initializeConfig() { if (this.scriptsDir === null) { // Handle both absolute and relative paths for testing @@ -63,7 +82,7 @@ export class ScriptManager { path: filePath, extension, size: stats.size, - lastModified: stats.mtime, + lastModified: this.safeMtime(stats.mtime), executable }); } @@ -125,7 +144,7 @@ export class ScriptManager { path: filePath, extension, size: stats.size, - lastModified: stats.mtime, + lastModified: this.safeMtime(stats.mtime), executable, logo, slug @@ -212,7 +231,7 @@ export class ScriptManager { path: filePath, extension, size: stats.size, - lastModified: stats.mtime, + lastModified: this.safeMtime(stats.mtime), executable, logo, slug diff --git a/src/server/services/autoSyncService.js b/src/server/services/autoSyncService.js index c3bf74a..f8372b8 100644 --- a/src/server/services/autoSyncService.js +++ b/src/server/services/autoSyncService.js @@ -1,5 +1,5 @@ import cron from 'node-cron'; -import { githubJsonService } from './githubJsonService.ts'; +import { githubJsonService } from './githubJsonService.js'; import { scriptDownloaderService } from './scriptDownloader.js'; import { appriseService } from './appriseService.js'; import { readFile, writeFile, readFileSync, writeFileSync } from 'fs'; @@ -12,6 +12,25 @@ export class AutoSyncService { this.isRunning = false; } + /** + * Safely convert a date to ISO string, handling invalid dates + * @param {Date} date - The date to convert + * @returns {string} - ISO string or fallback timestamp + */ + safeToISOString(date) { + try { + // Check if the date is valid + if (!date || isNaN(date.getTime())) { + console.warn('Invalid date provided to safeToISOString, using current time as fallback'); + return new Date().toISOString(); + } + return date.toISOString(); + } catch (error) { + console.warn('Error converting date to ISO string:', error instanceof Error ? error.message : String(error)); + return new Date().toISOString(); + } + } + /** * Load auto-sync settings from .env file */ @@ -251,56 +270,120 @@ export class AutoSyncService { const results = { jsonSync: syncResult, - newScripts: [], - updatedScripts: [], - errors: [] + newScripts: /** @type {string[]} */ ([]), + updatedScripts: /** @type {string[]} */ ([]), + errors: /** @type {string[]} */ ([]) }; // Step 2: Auto-download/update scripts if enabled const settings = this.loadSettings(); if (settings.autoDownloadNew || settings.autoUpdateExisting) { + console.log('Processing synced JSON files for script downloads...'); + // Only process scripts for files that were actually synced - // @ts-ignore - syncedFiles exists in the JavaScript version if (syncResult.syncedFiles && syncResult.syncedFiles.length > 0) { - // @ts-ignore - syncedFiles exists in the JavaScript version - console.log(`Processing ${syncResult.syncedFiles.length} synced JSON files for new scripts...`); + console.log(`Processing ${syncResult.syncedFiles.length} synced JSON files for script downloads...`); - // Get all scripts from synced files - // @ts-ignore - syncedFiles exists in the JavaScript version - const allSyncedScripts = await githubJsonService.getScriptsForFiles(syncResult.syncedFiles); + // Get scripts only for the synced files + const localScriptsService = await import('./localScripts.js'); + const syncedScripts = []; - // Initialize script downloader service - // @ts-ignore - initializeConfig is public in the JS version - scriptDownloaderService.initializeConfig(); - - // Filter to only truly NEW scripts (not previously downloaded) - const newScripts = []; - for (const script of allSyncedScripts) { - const isDownloaded = await scriptDownloaderService.isScriptDownloaded(script); - if (!isDownloaded) { - newScripts.push(script); + for (const filename of syncResult.syncedFiles) { + try { + // Extract slug from filename (remove .json extension) + const slug = filename.replace('.json', ''); + const script = await localScriptsService.localScriptsService.getScriptBySlug(slug); + if (script) { + syncedScripts.push(script); + } + } catch (error) { + console.warn(`Error loading script from ${filename}:`, error); } } - console.log(`Found ${newScripts.length} new scripts out of ${allSyncedScripts.length} total scripts`); + console.log(`Found ${syncedScripts.length} scripts from synced JSON files`); - if (settings.autoDownloadNew && newScripts.length > 0) { - console.log(`Auto-downloading ${newScripts.length} new scripts...`); - const downloadResult = await scriptDownloaderService.autoDownloadNewScripts(newScripts); - // @ts-ignore - Type assertion needed for dynamic assignment - results.newScripts = downloadResult.downloaded; - // @ts-ignore - Type assertion needed for dynamic assignment - results.errors.push(...downloadResult.errors); + // Filter to only truly NEW scripts (not previously downloaded) + const newScripts = []; + const existingScripts = []; + + for (const script of syncedScripts) { + try { + // Validate script object + if (!script || !script.slug) { + console.warn('Invalid script object found, skipping:', script); + continue; + } + + const isDownloaded = await scriptDownloaderService.isScriptDownloaded(script); + if (!isDownloaded) { + newScripts.push(script); + } else { + existingScripts.push(script); + } + } catch (error) { + console.warn(`Error checking script ${script?.slug || 'unknown'}:`, error); + // Treat as new script if we can't check + if (script && script.slug) { + newScripts.push(script); + } + } } - if (settings.autoUpdateExisting) { - console.log('Auto-updating existing scripts from synced files...'); - const updateResult = await scriptDownloaderService.autoUpdateExistingScripts(allSyncedScripts); - // @ts-ignore - Type assertion needed for dynamic assignment - results.updatedScripts = updateResult.updated; - // @ts-ignore - Type assertion needed for dynamic assignment - results.errors.push(...updateResult.errors); + console.log(`Found ${newScripts.length} new scripts and ${existingScripts.length} existing scripts from synced files`); + + // Download new scripts + if (settings.autoDownloadNew && newScripts.length > 0) { + console.log(`Auto-downloading ${newScripts.length} new scripts...`); + const downloaded = []; + const errors = []; + + for (const script of newScripts) { + try { + const result = await scriptDownloaderService.loadScript(script); + if (result.success) { + downloaded.push(script.name || script.slug); + console.log(`Downloaded script: ${script.name || script.slug}`); + } else { + errors.push(`${script.name || script.slug}: ${result.message}`); + } + } catch (error) { + const errorMsg = error instanceof Error ? error.message : String(error); + errors.push(`${script.name || script.slug}: ${errorMsg}`); + console.error(`Failed to download script ${script.slug}:`, error); + } + } + + results.newScripts = downloaded; + results.errors.push(...errors); + } + + // Update existing scripts + if (settings.autoUpdateExisting && existingScripts.length > 0) { + console.log(`Auto-updating ${existingScripts.length} existing scripts...`); + const updated = []; + const errors = []; + + for (const script of existingScripts) { + try { + // Always update existing scripts when auto-update is enabled + const result = await scriptDownloaderService.loadScript(script); + if (result.success) { + updated.push(script.name || script.slug); + console.log(`Updated script: ${script.name || script.slug}`); + } else { + errors.push(`${script.name || script.slug}: ${result.message}`); + } + } catch (error) { + const errorMsg = error instanceof Error ? error.message : String(error); + errors.push(`${script.name || script.slug}: ${errorMsg}`); + console.error(`Failed to update script ${script.slug}:`, error); + } + } + + results.updatedScripts = updated; + results.errors.push(...errors); } } else { console.log('No JSON files were synced, skipping script download/update'); @@ -316,7 +399,7 @@ export class AutoSyncService { } // Step 4: Update last sync time - const lastSyncTime = new Date().toISOString(); + const lastSyncTime = this.safeToISOString(new Date()); const updatedSettings = { ...settings, lastAutoSync: lastSyncTime }; this.saveSettings(updatedSettings); @@ -384,6 +467,12 @@ export class AutoSyncService { const grouped = new Map(); scripts.forEach(script => { + // Validate script object + if (!script || !script.name) { + console.warn('Invalid script object in groupScriptsByCategory, skipping:', script); + return; + } + const scriptCategories = script.categories || [0]; // Default to Miscellaneous (id: 0) scriptCategories.forEach((/** @type {number} */ catId) => { const categoryName = categoryMap.get(catId) || 'Miscellaneous'; diff --git a/src/server/services/githubJsonService.js b/src/server/services/githubJsonService.js index a5cdb6a..a3de782 100644 --- a/src/server/services/githubJsonService.js +++ b/src/server/services/githubJsonService.js @@ -1,276 +1,6 @@ -import { writeFile, mkdir } from 'fs/promises'; -import { readFileSync, readdirSync, statSync, utimesSync } from 'fs'; -import { join } from 'path'; -import { Buffer } from 'buffer'; +// JavaScript wrapper for githubJsonService.ts +// This allows the JavaScript autoSyncService.js to import the TypeScript service -export class GitHubJsonService { - constructor() { - this.baseUrl = null; - this.repoUrl = null; - this.branch = null; - this.jsonFolder = null; - this.localJsonDirectory = null; - this.scriptCache = new Map(); - } +import { githubJsonService } from './githubJsonService.ts'; - initializeConfig() { - if (this.repoUrl === null) { - // Get environment variables - this.repoUrl = process.env.REPO_URL || ""; - this.branch = process.env.REPO_BRANCH || "main"; - this.jsonFolder = process.env.JSON_FOLDER || "scripts"; - this.localJsonDirectory = join(process.cwd(), 'scripts', 'json'); - - // Only validate GitHub URL if it's provided - if (this.repoUrl) { - // Extract owner and repo from the URL - const urlMatch = /github\.com\/([^\/]+)\/([^\/]+)/.exec(this.repoUrl); - if (!urlMatch) { - throw new Error(`Invalid GitHub repository URL: ${this.repoUrl}`); - } - - const [, owner, repo] = urlMatch; - this.baseUrl = `https://api.github.com/repos/${owner}/${repo}`; - } else { - // Set a dummy base URL if no REPO_URL is provided - this.baseUrl = ""; - } - } - } - - async fetchFromGitHub(endpoint) { - this.initializeConfig(); - const response = await fetch(`${this.baseUrl}${endpoint}`, { - headers: { - 'Accept': 'application/vnd.github.v3+json', - 'User-Agent': 'PVEScripts-Local/1.0', - ...(process.env.GITHUB_TOKEN && { 'Authorization': `token ${process.env.GITHUB_TOKEN}` }) - }, - }); - - if (!response.ok) { - throw new Error(`GitHub API error: ${response.status} ${response.statusText}`); - } - - return response.json(); - } - - async syncJsonFiles() { - try { - this.initializeConfig(); - - if (!this.baseUrl) { - return { - success: false, - message: 'No GitHub repository configured' - }; - } - - console.log('Starting fast incremental JSON sync...'); - - // Ensure local directory exists - await mkdir(this.localJsonDirectory, { recursive: true }); - - // Step 1: Get file list from GitHub (single API call) - console.log('Fetching file list from GitHub...'); - const files = await this.fetchFromGitHub(`/contents/${this.jsonFolder}?ref=${this.branch}`); - - if (!Array.isArray(files)) { - throw new Error('Invalid response from GitHub API'); - } - - const jsonFiles = files.filter(file => file.name.endsWith('.json')); - console.log(`Found ${jsonFiles.length} JSON files in repository`); - - // Step 2: Get local file list (fast local operation) - const localFiles = new Map(); - try { - console.log(`Looking for local files in: ${this.localJsonDirectory}`); - const localFileList = readdirSync(this.localJsonDirectory); - console.log(`Found ${localFileList.length} files in local directory`); - for (const fileName of localFileList) { - if (fileName.endsWith('.json')) { - const filePath = join(this.localJsonDirectory, fileName); - const stats = statSync(filePath); - localFiles.set(fileName, { - mtime: stats.mtime, - size: stats.size - }); - } - } - } catch (error) { - console.log('Error reading local directory:', error.message); - console.log('Directory path:', this.localJsonDirectory); - console.log('No local files found, will download all'); - } - - console.log(`Found ${localFiles.size} local JSON files`); - - // Step 3: Compare and identify files that need syncing - const filesToSync = []; - let skippedCount = 0; - - for (const file of jsonFiles) { - const localFile = localFiles.get(file.name); - - if (!localFile) { - // File doesn't exist locally - filesToSync.push(file); - console.log(`Missing: ${file.name}`); - } else { - // Compare modification times and sizes - const localMtime = new Date(localFile.mtime); - const remoteMtime = new Date(file.updated_at); - const localSize = localFile.size; - const remoteSize = file.size; - - // Sync if remote is newer OR sizes are different (content changed) - if (localMtime < remoteMtime || localSize !== remoteSize) { - filesToSync.push(file); - console.log(`Changed: ${file.name} (${localMtime.toISOString()} -> ${remoteMtime.toISOString()})`); - } else { - skippedCount++; - console.log(`Up-to-date: ${file.name}`); - } - } - } - - console.log(`Files to sync: ${filesToSync.length}, Up-to-date: ${skippedCount}`); - - // Step 4: Download only the files that need syncing - let syncedCount = 0; - const errors = []; - const syncedFiles = []; - - // Process files in batches to avoid overwhelming the API - const batchSize = 10; - for (let i = 0; i < filesToSync.length; i += batchSize) { - const batch = filesToSync.slice(i, i + batchSize); - - // Process batch in parallel - const promises = batch.map(async (file) => { - try { - const content = await this.fetchFromGitHub(`/contents/${file.path}?ref=${this.branch}`); - - if (content.content) { - // Decode base64 content - const fileContent = Buffer.from(content.content, 'base64').toString('utf-8'); - - // Write to local file - const localPath = join(this.localJsonDirectory, file.name); - await writeFile(localPath, fileContent, 'utf-8'); - - // Update file modification time to match remote - const remoteMtime = new Date(file.updated_at); - utimesSync(localPath, remoteMtime, remoteMtime); - - syncedCount++; - syncedFiles.push(file.name); - console.log(`Synced: ${file.name}`); - } - } catch (error) { - console.error(`Failed to sync ${file.name}:`, error.message); - errors.push(`${file.name}: ${error.message}`); - } - }); - - await Promise.all(promises); - - // Small delay between batches to be nice to the API - if (i + batchSize < filesToSync.length) { - await new Promise(resolve => setTimeout(resolve, 100)); - } - } - - console.log(`JSON sync completed. Synced ${syncedCount} files, skipped ${skippedCount} files.`); - - return { - success: true, - message: `Successfully synced ${syncedCount} JSON files (${skippedCount} up-to-date)`, - syncedCount, - skippedCount, - syncedFiles, - errors - }; - - } catch (error) { - console.error('JSON sync failed:', error); - return { - success: false, - message: error.message, - error: error.message - }; - } - } - - async getAllScripts() { - try { - this.initializeConfig(); - - if (!this.localJsonDirectory) { - return []; - } - - const scripts = []; - - // Read all JSON files from local directory - const files = readdirSync(this.localJsonDirectory); - const jsonFiles = files.filter(file => file.endsWith('.json')); - - for (const file of jsonFiles) { - try { - const filePath = join(this.localJsonDirectory, file); - const content = readFileSync(filePath, 'utf-8'); - const script = JSON.parse(content); - - if (script && typeof script === 'object') { - scripts.push(script); - } - } catch (error) { - console.error(`Failed to parse ${file}:`, error.message); - } - } - - return scripts; - } catch (error) { - console.error('Failed to get all scripts:', error); - return []; - } - } - - /** - * Get scripts only for specific JSON files that were synced - */ - async getScriptsForFiles(syncedFiles) { - try { - this.initializeConfig(); - - if (!this.localJsonDirectory || !syncedFiles || syncedFiles.length === 0) { - return []; - } - - const scripts = []; - - for (const fileName of syncedFiles) { - try { - const filePath = join(this.localJsonDirectory, fileName); - const content = readFileSync(filePath, 'utf-8'); - const script = JSON.parse(content); - - if (script && typeof script === 'object') { - scripts.push(script); - } - } catch (error) { - console.error(`Failed to parse ${fileName}:`, error.message); - } - } - - return scripts; - } catch (error) { - console.error('Failed to get scripts for synced files:', error); - return []; - } - } -} - -export const githubJsonService = new GitHubJsonService(); +export { githubJsonService }; diff --git a/src/server/services/githubJsonService.ts b/src/server/services/githubJsonService.ts index 344d51e..30e0742 100644 --- a/src/server/services/githubJsonService.ts +++ b/src/server/services/githubJsonService.ts @@ -1,4 +1,4 @@ -import { writeFile, mkdir } from 'fs/promises'; +import { writeFile, mkdir, readdir } from 'fs/promises'; import { join } from 'path'; import { env } from '../../env.js'; import type { Script, ScriptCard, GitHubFile } from '../../types/script'; @@ -185,48 +185,90 @@ export class GitHubJsonService { } } - async syncJsonFiles(): Promise<{ success: boolean; message: string; count: number }> { + async syncJsonFiles(): Promise<{ success: boolean; message: string; count: number; syncedFiles: string[] }> { try { - // Get all scripts from GitHub (1 API call + raw downloads) - const scripts = await this.getAllScripts(); + console.log('Starting fast incremental JSON sync...'); - // Save scripts to local directory - await this.saveScriptsLocally(scripts); + // Get file list from GitHub + console.log('Fetching file list from GitHub...'); + const githubFiles = await this.getJsonFiles(); + console.log(`Found ${githubFiles.length} JSON files in repository`); + + // Get local files + const localFiles = await this.getLocalJsonFiles(); + console.log(`Found ${localFiles.length} files in local directory`); + console.log(`Found ${localFiles.filter(f => f.endsWith('.json')).length} local JSON files`); + + // Compare and find files that need syncing + const filesToSync = this.findFilesToSync(githubFiles, localFiles); + console.log(`Found ${filesToSync.length} files that need syncing`); + + if (filesToSync.length === 0) { + return { + success: true, + message: 'All JSON files are up to date', + count: 0, + syncedFiles: [] + }; + } + + // Download and save only the files that need syncing + const syncedFiles = await this.syncSpecificFiles(filesToSync); return { success: true, - message: `Successfully synced ${scripts.length} scripts from GitHub using 1 API call + raw downloads`, - count: scripts.length + message: `Successfully synced ${syncedFiles.length} JSON files from GitHub`, + count: syncedFiles.length, + syncedFiles }; } catch (error) { - console.error('Error syncing JSON files:', error); + console.error('JSON sync failed:', error); return { success: false, message: `Failed to sync JSON files: ${error instanceof Error ? error.message : 'Unknown error'}`, - count: 0 + count: 0, + syncedFiles: [] }; } } - private async saveScriptsLocally(scripts: Script[]): Promise { + private async getLocalJsonFiles(): Promise { this.initializeConfig(); try { - // Ensure the directory exists - await mkdir(this.localJsonDirectory!, { recursive: true }); - - // Save each script as a JSON file - for (const script of scripts) { - const filename = `${script.slug}.json`; - const filePath = join(this.localJsonDirectory!, filename); - const content = JSON.stringify(script, null, 2); - await writeFile(filePath, content, 'utf-8'); - } - - } catch (error) { - console.error('Error saving scripts locally:', error); - throw new Error('Failed to save scripts locally'); + const files = await readdir(this.localJsonDirectory!); + return files.filter(f => f.endsWith('.json')); + } catch { + return []; } } + + private findFilesToSync(githubFiles: GitHubFile[], localFiles: string[]): GitHubFile[] { + const localFileSet = new Set(localFiles); + // Return only files that don't exist locally + return githubFiles.filter(ghFile => !localFileSet.has(ghFile.name)); + } + + private async syncSpecificFiles(filesToSync: GitHubFile[]): Promise { + this.initializeConfig(); + const syncedFiles: string[] = []; + + await mkdir(this.localJsonDirectory!, { recursive: true }); + + for (const file of filesToSync) { + try { + const script = await this.downloadJsonFile(file.path); + const filename = `${script.slug}.json`; + const filePath = join(this.localJsonDirectory!, filename); + await writeFile(filePath, JSON.stringify(script, null, 2), 'utf-8'); + syncedFiles.push(filename); + } catch (error) { + console.error(`Failed to sync ${file.name}:`, error); + } + } + + return syncedFiles; + } + } // Singleton instance diff --git a/src/server/services/localScripts.js b/src/server/services/localScripts.js new file mode 100644 index 0000000..410ab48 --- /dev/null +++ b/src/server/services/localScripts.js @@ -0,0 +1,6 @@ +// JavaScript wrapper for localScripts.ts +// This allows the JavaScript autoSyncService.js to import the TypeScript service + +import { localScriptsService } from './localScripts.ts'; + +export { localScriptsService }; diff --git a/src/server/services/scriptDownloader.js b/src/server/services/scriptDownloader.js index fd06b8a..5bb69d2 100644 --- a/src/server/services/scriptDownloader.js +++ b/src/server/services/scriptDownloader.js @@ -1,14 +1,18 @@ -import { writeFile, readFile, mkdir } from 'fs/promises'; +// Real JavaScript implementation for script downloading import { join } from 'path'; +import { writeFile, mkdir } from 'fs/promises'; export class ScriptDownloaderService { constructor() { this.scriptsDirectory = null; + this.repoUrl = null; } initializeConfig() { if (this.scriptsDirectory === null) { this.scriptsDirectory = join(process.cwd(), 'scripts'); + // Get REPO_URL from environment or use default + this.repoUrl = process.env.REPO_URL || 'https://github.com/community-scripts/ProxmoxVE'; } } @@ -23,14 +27,27 @@ export class ScriptDownloaderService { } async downloadFileFromGitHub(filePath) { - // This is a simplified version - in a real implementation, - // you would fetch the file content from GitHub - // For now, we'll return a placeholder - return `#!/bin/bash -# Downloaded script: ${filePath} -# This is a placeholder - implement actual GitHub file download -echo "Script downloaded: ${filePath}" -`; + this.initializeConfig(); + if (!this.repoUrl) { + throw new Error('REPO_URL environment variable is not set'); + } + + // Extract repo path from URL + const match = /github\.com\/([^\/]+)\/([^\/]+)/.exec(this.repoUrl); + if (!match) { + throw new Error('Invalid GitHub repository URL'); + } + const [, owner, repo] = match; + + const url = `https://raw.githubusercontent.com/${owner}/${repo}/main/${filePath}`; + + console.log(`Downloading from GitHub: ${url}`); + const response = await fetch(url); + if (!response.ok) { + throw new Error(`Failed to download ${filePath}: ${response.status} ${response.statusText}`); + } + + return response.text(); } modifyScriptContent(content) { @@ -57,6 +74,7 @@ echo "Script downloaded: ${filePath}" if (fileName) { // Download from GitHub + console.log(`Downloading script file: ${scriptPath}`); const content = await this.downloadFileFromGitHub(scriptPath); // Determine target directory based on script path @@ -111,6 +129,7 @@ echo "Script downloaded: ${filePath}" } files.push(`${finalTargetDir}/${fileName}`); + console.log(`Successfully downloaded: ${finalTargetDir}/${fileName}`); } } } @@ -121,12 +140,15 @@ echo "Script downloaded: ${filePath}" if (hasCtScript) { const installScriptName = `${script.slug}-install.sh`; try { + console.log(`Downloading install script: install/${installScriptName}`); const installContent = await this.downloadFileFromGitHub(`install/${installScriptName}`); const localInstallPath = join(this.scriptsDirectory, 'install', installScriptName); await writeFile(localInstallPath, installContent, 'utf-8'); files.push(`install/${installScriptName}`); - } catch { + console.log(`Successfully downloaded: install/${installScriptName}`); + } catch (error) { // Install script might not exist, that's okay + console.log(`Install script not found: install/${installScriptName}`); } } @@ -145,78 +167,6 @@ echo "Script downloaded: ${filePath}" } } - /** - * Auto-download new scripts that haven't been downloaded yet - */ - async autoDownloadNewScripts(allScripts) { - this.initializeConfig(); - const downloaded = []; - const errors = []; - - for (const script of allScripts) { - try { - // Check if script is already downloaded - const isDownloaded = await this.isScriptDownloaded(script); - - if (!isDownloaded) { - const result = await this.loadScript(script); - if (result.success) { - downloaded.push(script); // Return full script object instead of just name - console.log(`Auto-downloaded new script: ${script.name || script.slug}`); - } else { - errors.push(`${script.name || script.slug}: ${result.message}`); - } - } - } catch (error) { - const errorMsg = `${script.name || script.slug}: ${error instanceof Error ? error.message : 'Unknown error'}`; - errors.push(errorMsg); - console.error(`Failed to auto-download script ${script.slug}:`, error); - } - } - - return { downloaded, errors }; - } - - /** - * Auto-update existing scripts to newer versions - */ - async autoUpdateExistingScripts(allScripts) { - this.initializeConfig(); - const updated = []; - const errors = []; - - for (const script of allScripts) { - try { - // Check if script is downloaded - const isDownloaded = await this.isScriptDownloaded(script); - - if (isDownloaded) { - // Check if update is needed by comparing content - const needsUpdate = await this.scriptNeedsUpdate(script); - - if (needsUpdate) { - const result = await this.loadScript(script); - if (result.success) { - updated.push(script); // Return full script object instead of just name - console.log(`Auto-updated script: ${script.name || script.slug}`); - } else { - errors.push(`${script.name || script.slug}: ${result.message}`); - } - } - } - } catch (error) { - const errorMsg = `${script.name || script.slug}: ${error instanceof Error ? error.message : 'Unknown error'}`; - errors.push(errorMsg); - console.error(`Failed to auto-update script ${script.slug}:`, error); - } - } - - return { updated, errors }; - } - - /** - * Check if a script is already downloaded - */ async isScriptDownloaded(script) { if (!script.install_methods?.length) return false; @@ -261,7 +211,7 @@ echo "Script downloaded: ${filePath}" } try { - await readFile(filePath, 'utf8'); + await import('fs/promises').then(fs => fs.readFile(filePath, 'utf8')); // File exists, continue checking other methods } catch { // File doesn't exist, script is not fully downloaded @@ -274,73 +224,6 @@ echo "Script downloaded: ${filePath}" // All files exist, script is downloaded return true; } - - /** - * Check if a script needs updating by comparing local and remote content - */ - async scriptNeedsUpdate(script) { - if (!script.install_methods?.length) return false; - - for (const method of script.install_methods) { - if (method.script) { - const scriptPath = method.script; - const fileName = scriptPath.split('/').pop(); - - if (fileName) { - // Determine target directory based on script path - let targetDir; - let finalTargetDir; - let filePath; - - if (scriptPath.startsWith('ct/')) { - targetDir = 'ct'; - finalTargetDir = targetDir; - filePath = join(this.scriptsDirectory, targetDir, fileName); - } else if (scriptPath.startsWith('tools/')) { - targetDir = 'tools'; - const subPath = scriptPath.replace('tools/', ''); - const subDir = subPath.includes('/') ? subPath.substring(0, subPath.lastIndexOf('/')) : ''; - finalTargetDir = subDir ? join(targetDir, subDir) : targetDir; - filePath = join(this.scriptsDirectory, finalTargetDir, fileName); - } else if (scriptPath.startsWith('vm/')) { - targetDir = 'vm'; - const subPath = scriptPath.replace('vm/', ''); - const subDir = subPath.includes('/') ? subPath.substring(0, subPath.lastIndexOf('/')) : ''; - finalTargetDir = subDir ? join(targetDir, subDir) : targetDir; - filePath = join(this.scriptsDirectory, finalTargetDir, fileName); - } else if (scriptPath.startsWith('vw/')) { - targetDir = 'vw'; - const subPath = scriptPath.replace('vw/', ''); - const subDir = subPath.includes('/') ? subPath.substring(0, subPath.lastIndexOf('/')) : ''; - finalTargetDir = subDir ? join(targetDir, subDir) : targetDir; - filePath = join(this.scriptsDirectory, finalTargetDir, fileName); - } else { - targetDir = 'ct'; - finalTargetDir = targetDir; - filePath = join(this.scriptsDirectory, targetDir, fileName); - } - - try { - // Read local content - const localContent = await readFile(filePath, 'utf8'); - - // Download remote content - const remoteContent = await this.downloadFileFromGitHub(scriptPath); - - // Compare content (simple string comparison for now) - // In a more sophisticated implementation, you might want to compare - // file modification times or use content hashing - return localContent !== remoteContent; - } catch { - // If we can't read local or download remote, assume update needed - return true; - } - } - } - } - - return false; - } } export const scriptDownloaderService = new ScriptDownloaderService();