From 7fd314fe1793aa66ff35e9916ff1442104be75bf Mon Sep 17 00:00:00 2001 From: Gordon Mickel Date: Sun, 21 Jul 2024 22:19:26 +0200 Subject: [PATCH 1/3] feat: add line-numbers option --- src/cli/index.ts | 3 + src/cli/interactive-filtering.ts | 2 + src/core/markdown-generator.ts | 23 +++++-- src/utils/file-cache.ts | 34 +++++++--- tests/e2e/cli-commands.test.ts | 65 +++++++++++++++++++- tests/fixtures/test-project/src/test-file.js | 3 + tests/unit/markdown-generator.test.ts | 32 ++++++++++ 7 files changed, 147 insertions(+), 15 deletions(-) create mode 100644 tests/fixtures/test-project/src/test-file.js diff --git a/src/cli/index.ts b/src/cli/index.ts index dabd883..fb07940 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -41,6 +41,7 @@ export function cli(args: string[]) { 'File patterns to exclude (use glob patterns, e.g., "**/*.test.js")', ) .option('-s, --suppress-comments', 'Strip comments from the code') + .option('-l, --line-numbers', 'Add line numbers to code blocks') .option('--case-sensitive', 'Use case-sensitive pattern matching') .option( '--no-codeblock', @@ -92,6 +93,7 @@ export function cli(args: string[]) { noCodeblock: !options.codeblock, basePath: options.path, customData, + lineNumbers: options.lineNumbers, }); if (options.prompt) { @@ -132,6 +134,7 @@ export function cli(args: string[]) { 'File patterns to exclude (use glob patterns, e.g., "**/*.test.js")', ) .option('-s, --suppress-comments', 'Strip comments from the code') + .option('-l, --line-numbers', 'Add line numbers to code blocks') .option('--case-sensitive', 'Use case-sensitive pattern matching') .option( '--no-codeblock', diff --git a/src/cli/interactive-filtering.ts b/src/cli/interactive-filtering.ts index 9e8cb89..0dfd69d 100644 --- a/src/cli/interactive-filtering.ts +++ b/src/cli/interactive-filtering.ts @@ -33,6 +33,7 @@ interface InteractiveModeOptions { cachePath?: string; respectGitignore?: boolean; invert?: boolean; + lineNumbers?: boolean; } export async function interactiveMode(options: InteractiveModeOptions) { @@ -82,6 +83,7 @@ export async function interactiveMode(options: InteractiveModeOptions) { customData: options.customData ? JSON.parse(options.customData) : undefined, + lineNumbers: options.lineNumbers, }; let markdown = await generateMarkdown( diff --git a/src/core/markdown-generator.ts b/src/core/markdown-generator.ts index 63a4625..2ceaba6 100644 --- a/src/core/markdown-generator.ts +++ b/src/core/markdown-generator.ts @@ -7,16 +7,29 @@ export interface MarkdownOptions { noCodeblock?: boolean; customData?: Record; basePath?: string; + lineNumbers?: boolean; } -function registerHandlebarsHelpers(noCodeblock: boolean) { +function registerHandlebarsHelpers( + noCodeblock: boolean, + options: MarkdownOptions, +) { Handlebars.registerHelper( 'codeblock', (content: string, language: string) => { - if (noCodeblock) { - return content; + let numberedContent = content; + if (options.lineNumbers) { + numberedContent = content + .split('\n') + .map((line, index) => `${index + 1} ${line}`) + .join('\n'); } - return new Handlebars.SafeString(`\`\`\`${language}\n${content}\n\`\`\``); + if (options.noCodeblock) { + return numberedContent; + } + return new Handlebars.SafeString( + `\`\`\`${language}\n${numberedContent}\n\`\`\``, + ); }, ); @@ -87,7 +100,7 @@ export async function generateMarkdown( basePath = process.cwd(), } = options; - registerHandlebarsHelpers(noCodeblock); + registerHandlebarsHelpers(noCodeblock, options); const compiledTemplate = Handlebars.compile(templateContent); diff --git a/src/utils/file-cache.ts b/src/utils/file-cache.ts index 1a2e1b8..1ea01d0 100644 --- a/src/utils/file-cache.ts +++ b/src/utils/file-cache.ts @@ -8,6 +8,8 @@ interface CacheEntry { data: FileInfo; } +const MAX_CACHE_ITEM_SIZE = 1024 * 1024; // 1 MB + export class FileCache { private cacheFile: string; private cache: Record = {}; @@ -69,16 +71,22 @@ export class FileCache { try { await fs.ensureDir(path.dirname(this.cacheFile)); const tempFile = `${this.cacheFile}.tmp`; - await fs.writeFile( - tempFile, - JSON.stringify(this.cache, (key, value) => { - if (value instanceof Date) { - return value.toISOString(); + + // Stringify cache items individually + const cacheEntries = Object.entries(this.cache) + .map(([key, value]) => { + try { + return `"${key}":${JSON.stringify(value)}`; + } catch (error) { + console.warn(`Failed to stringify cache entry for ${key}:`, error); + return null; } - return value; - }), - 'utf-8', - ); + }) + .filter(Boolean); + + const cacheString = `{${cacheEntries.join(',')}}`; + + await fs.writeFile(tempFile, cacheString, 'utf-8'); await fs.rename(tempFile, this.cacheFile); this.isDirty = false; } catch (error) { @@ -103,6 +111,14 @@ export class FileCache { async set(filePath: string, data: FileInfo): Promise { await this.loadCache(); const hash = await this.calculateFileHash(filePath); + + // Check the size of the data + const dataSize = JSON.stringify(data).length; + if (dataSize > MAX_CACHE_ITEM_SIZE) { + console.warn(`Skipping cache for large file: ${filePath}`); + return; + } + this.cache[filePath] = { hash, data }; this.isDirty = true; } diff --git a/tests/e2e/cli-commands.test.ts b/tests/e2e/cli-commands.test.ts index 344db57..44a7c2e 100644 --- a/tests/e2e/cli-commands.test.ts +++ b/tests/e2e/cli-commands.test.ts @@ -1,9 +1,13 @@ -import { execSync } from 'node:child_process'; +import { exec, execSync } from 'node:child_process'; +import os from 'node:os'; import path from 'node:path'; +import { promisify } from 'node:util'; import fs from 'fs-extra'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; import { normalizePath } from '../../src/utils/normalize-path'; +const execAsync = promisify(exec); + describe('CLI Commands', () => { const cliPath = path.resolve(__dirname, '../../cli.js'); const testProjectPath = path.resolve(__dirname, '../fixtures/test-project'); @@ -172,6 +176,7 @@ describe('CLI Commands', () => { fs.removeSync(customCachePath); } }); + it('should generate markdown with custom data and prompt', () => { const customData = JSON.stringify({ projectName: 'My Awesome Project', @@ -217,4 +222,62 @@ describe('CLI Commands', () => { expect(output).toContain('## Your Task'); expect(output).toContain('Please review this code and provide feedback.'); }); + + it('should generate markdown with line numbers when --line-numbers flag is used', async () => { + const testDir1 = path.join(os.tmpdir(), 'test-project-1'); + const testDir2 = path.join(os.tmpdir(), 'test-project-2'); + + await fs.ensureDir(testDir1); + await fs.ensureDir(testDir2); + + const testFile1 = path.join(testDir1, 'test-file.js'); + const testFile2 = path.join(testDir2, 'test-file.js'); + + await fs.writeFile( + testFile1, + 'const x = 1;\nconst y = 2;\nconsole.log(x + y);', + ); + await fs.writeFile( + testFile2, + 'const x = 1;\nconst y = 2;\nconsole.log(x + y);', + ); + + const outputPath1 = path.join(testDir1, 'output-with-line-numbers.md'); + const outputPath2 = path.join(testDir2, 'output-without-line-numbers.md'); + + const commandWithLineNumbers = `pnpm exec esno ${cliPath} generate -p "${normalizePath(testDir1)}" -o "${normalizePath(outputPath1)}" --line-numbers`; + const commandWithoutLineNumbers = `pnpm exec esno ${cliPath} generate -p "${normalizePath(testDir2)}" -o "${normalizePath(outputPath2)}"`; + + try { + await execAsync(commandWithLineNumbers, { + env: { ...process.env, NODE_ENV: 'test' }, + cwd: path.resolve(__dirname, '../..'), + }); + + // Add a small delay to ensure file system operations are complete + await new Promise((resolve) => setTimeout(resolve, 1000)); + + await execAsync(commandWithoutLineNumbers, { + env: { ...process.env, NODE_ENV: 'test' }, + cwd: path.resolve(__dirname, '../..'), + }); + + const outputWithLineNumbers = await fs.readFile(outputPath1, 'utf-8'); + const outputWithoutLineNumbers = await fs.readFile(outputPath2, 'utf-8'); + + expect(outputWithLineNumbers).toContain('1 const x = 1;'); + expect(outputWithLineNumbers).toContain('2 const y = 2;'); + expect(outputWithLineNumbers).toContain('3 console.log(x + y);'); + + expect(outputWithoutLineNumbers).not.toContain('1 const x = 1;'); + expect(outputWithoutLineNumbers).toContain('const x = 1;'); + } catch (error) { + console.error('Error executing commands:', error); + throw error; + } finally { + // Clean up + await fs.remove(testDir1); + await fs.remove(testDir2); + } + }); }); diff --git a/tests/fixtures/test-project/src/test-file.js b/tests/fixtures/test-project/src/test-file.js new file mode 100644 index 0000000..253b61b --- /dev/null +++ b/tests/fixtures/test-project/src/test-file.js @@ -0,0 +1,3 @@ +const x = 1; +const y = 2; +console.log(x + y); \ No newline at end of file diff --git a/tests/unit/markdown-generator.test.ts b/tests/unit/markdown-generator.test.ts index 87c7fa9..166edbb 100644 --- a/tests/unit/markdown-generator.test.ts +++ b/tests/unit/markdown-generator.test.ts @@ -188,4 +188,36 @@ describe('Markdown Generator', () => { 'Please review this code and provide feedback.', ); }); + + it('should add line numbers to code blocks when lineNumbers option is true', async () => { + const mockFiles = [ + { + path: '/project/src/index.ts', + extension: 'ts', + language: 'typescript', + size: 100, + created: new Date('2023-01-01'), + modified: new Date('2023-01-02'), + content: 'const greeting = "Hello";\nconsole.log(greeting);', + }, + ]; + + const template = + '{{#each files}}{{#codeblock this.content this.language}}{{/codeblock}}{{/each}}'; + + const resultWithoutLineNumbers = await generateMarkdown( + mockFiles, + template, + { lineNumbers: false }, + ); + const resultWithLineNumbers = await generateMarkdown(mockFiles, template, { + lineNumbers: true, + }); + + expect(resultWithoutLineNumbers).not.toContain('1 const'); + expect(resultWithoutLineNumbers).not.toContain('2 console'); + + expect(resultWithLineNumbers).toContain('1 const greeting = "Hello";'); + expect(resultWithLineNumbers).toContain('2 console.log(greeting);'); + }); }); From cb9609dd0037ba421c298709f53e41279f88fb7a Mon Sep 17 00:00:00 2001 From: Gordon Mickel Date: Sun, 21 Jul 2024 22:19:34 +0200 Subject: [PATCH 2/3] chore: update README --- README.md | 420 ++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 411 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 78fe15d..c7d06e9 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,6 @@ -# code-whisper +# CodeWhisper + +![CodeWhisper logo](/api/placeholder/200/200) add blazing fast AI-friendly prompts to your codebase @@ -8,25 +10,392 @@ add blazing fast AI-friendly prompts to your codebase [![GitHub Stars](https://img.shields.io/github/stars/gmickel/CodeWhisper.svg)](https://github.com/gmickel/CodeWhisper/stargazers) [![GitHub Forks](https://img.shields.io/github/forks/gmickel/CodeWhisper.svg)](https://github.com/gmickel/CodeWhisper/network) -_description_ +[Key Features](#-key-features) • +[Quick Start](#-quick-start) • +[Installation](#-installation) • +[Usage](#-usage) • +[API](#-api) • +[Contributing](#-contributing) • +[License](#-license) + +## 📖 About + +CodeWhisper is a powerful tool designed to convert your repository code into AI-friendly prompts. It streamlines the process of generating comprehensive code summaries, making it easier to integrate your codebase with AI-powered tools and workflows. + +## ✨ Key Features + +* 🚀 Blazingly fast code processing with concurrent workers +* 🎯 Customizable file filtering and exclusion +* 📊 Intelligent caching for improved performance +* 🔧 Extensible template system +* 🖥️ CLI and programmatic API +* 🔒 Respects .gitignore rules +* 🌈 Full language support for all text-based file types +* 🤖 Interactive mode for granular file selection +* ⚡ Optimized for large repositories + +## 🚀 Quick Start + +```bash +# Install CodeWhisper globally +npm install -g codewhisper + +# Navigate to your project directory +cd /path/to/your/project + +# Generate an AI-friendly prompt +codewhisper generate +``` + +## 📦 Installation + +You can install CodeWhisper using your preferred package manager: + +```bash +# Using npm +npm install -g codewhisper + +# Using yarn +yarn global add codewhisper + +# Using pnpm +pnpm add -g codewhisper + +# Using bun +bun add -g codewhisper +``` + +## 💻 Usage + +### Basic Usage + +Generate a markdown file from your codebase: + +```bash +codewhisper generate -p /path/to/your/project -o output.md +``` + +### Default Ignores + +CodeWhisper comes with a set of default ignore patterns to exclude common files and directories that are typically not needed in code analysis (e.g., `.git` , `node_modules` , build outputs, etc.). You can view the full list of default ignores in the [file-processor.ts](https://github.com/gmickel/CodeWhisper/blob/main/src/core/file-processor.ts#L41) file. + +These default ignores can be overridden using the filter options or interactive selection. For example, if you want to include files that are ignored by default, you can use the `-f` or `--filter` option: + +```bash +codewhisper generate -f "node_modules/**/*.js" +``` + +Alternatively, you can use the interactive mode to manually select files and directories, including those that would be ignored by default: + +```bash +codewhisper interactive +``` + +### Binary Files + +CodeWhisper ignores all binary files by default. + +### Advanced Options + +* `-p, --path `: Path to the codebase (default: current directory) +* `-o, --output `: Output file name +* `-t, --template