Add ESLint configuration, .gitignore, Deno configuration, and initial Jupyter notebook
This commit is contained in:
12
.eslintrc
Normal file
12
.eslintrc
Normal file
@@ -0,0 +1,12 @@
|
|||||||
|
{
|
||||||
|
"rules": {
|
||||||
|
"@typescript-eslint/no-unused-vars": [
|
||||||
|
"error",
|
||||||
|
{
|
||||||
|
"argsIgnorePattern": "^_",
|
||||||
|
"varsIgnorePattern": "^_",
|
||||||
|
"caughtErrorsIgnorePattern": "^_"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
3
.gitignore
vendored
Normal file
3
.gitignore
vendored
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
.env
|
||||||
|
books/*
|
||||||
|
mindmap/*
|
||||||
6
deno.json
Normal file
6
deno.json
Normal file
@@ -0,0 +1,6 @@
|
|||||||
|
{
|
||||||
|
"imports": {
|
||||||
|
"@openai/openai": "jsr:@openai/openai@^6.17.0",
|
||||||
|
"@std/path": "jsr:@std/path@^1.1.4"
|
||||||
|
}
|
||||||
|
}
|
||||||
55
deno.lock
generated
Normal file
55
deno.lock
generated
Normal file
@@ -0,0 +1,55 @@
|
|||||||
|
{
|
||||||
|
"version": "5",
|
||||||
|
"specifiers": {
|
||||||
|
"jsr:@openai/openai@*": "6.17.0",
|
||||||
|
"jsr:@openai/openai@^6.17.0": "6.17.0",
|
||||||
|
"jsr:@std/dotenv@*": "0.225.6",
|
||||||
|
"jsr:@std/internal@^1.0.12": "1.0.12",
|
||||||
|
"jsr:@std/path@*": "1.1.4",
|
||||||
|
"jsr:@std/path@^1.1.4": "1.1.4",
|
||||||
|
"npm:fs@*": "0.0.1-security",
|
||||||
|
"npm:zod@3": "3.25.76"
|
||||||
|
},
|
||||||
|
"jsr": {
|
||||||
|
"@openai/openai@6.17.0": {
|
||||||
|
"integrity": "0e8947550cea8759af370b3ed3c646a88624054f397fb84c9bf76e843c580824",
|
||||||
|
"dependencies": [
|
||||||
|
"npm:zod"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"@std/dotenv@0.225.6": {
|
||||||
|
"integrity": "1d6f9db72f565bd26790fa034c26e45ecb260b5245417be76c2279e5734c421b"
|
||||||
|
},
|
||||||
|
"@std/internal@1.0.12": {
|
||||||
|
"integrity": "972a634fd5bc34b242024402972cd5143eac68d8dffaca5eaa4dba30ce17b027"
|
||||||
|
},
|
||||||
|
"@std/path@1.1.4": {
|
||||||
|
"integrity": "1d2d43f39efb1b42f0b1882a25486647cb851481862dc7313390b2bb044314b5",
|
||||||
|
"dependencies": [
|
||||||
|
"jsr:@std/internal"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"npm": {
|
||||||
|
"fs@0.0.1-security": {
|
||||||
|
"integrity": "sha512-3XY9e1pP0CVEUCdj5BmfIZxRBTSDycnbqhIOGec9QYtmVH2fbLpj86CFWkrNOkt/Fvty4KZG5lTglL9j/gJ87w=="
|
||||||
|
},
|
||||||
|
"zod@3.25.76": {
|
||||||
|
"integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ=="
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"remote": {
|
||||||
|
"https://cdn.skypack.dev/-/@pdf-lib/standard-fonts@v1.0.0-Z8Dxw2SAoP39vRoTIKF1/dist=es2019,mode=imports/optimized/@pdf-lib/standard-fonts.js": "7e7233fe50eff7cdacd5dfe148b2e8793ef2c0aacd1b09d09d74fa71ea588eac",
|
||||||
|
"https://cdn.skypack.dev/-/@pdf-lib/upng@v1.0.1-cDyYONcMEAs6JipZyc8F/dist=es2019,mode=imports/optimized/@pdf-lib/upng.js": "c1fa7f102bc8ece5a18e22f27a18929143c4bba9735b24ea23b0b0d8eaa8cd53",
|
||||||
|
"https://cdn.skypack.dev/-/pako@v1.0.11-HN3j6r6YXxSYMQQ5YY76/dist=es2019,mode=imports/optimized/pako.js": "cb928ba393bdb8c8b9b6509a68975a00e43780ed2424bf0c6e020890dc546e4b",
|
||||||
|
"https://cdn.skypack.dev/-/pdf-lib@v1.17.1-P8YmSI9gov9vMnrV7v2f/dist=es2019,mode=imports/optimized/pdf-lib.js": "ce55a997148b41a520719255d57af560fd269685241031d2502f0c7583d690b2",
|
||||||
|
"https://cdn.skypack.dev/-/tslib@v1.14.1-mWTe8wftGbG1xYC8gPyw/dist=es2019,mode=imports/optimized/tslib.js": "bde9e42ab1dee8764c8fef53a8165ab516af507bd5a6e120aae6d0922925ba74",
|
||||||
|
"https://cdn.skypack.dev/pdf-lib@^1.11.1?dts": "11ccb0b2660b159faec9facfc77e47ad78e1d1ee41805f007b4dd11b62095e05"
|
||||||
|
},
|
||||||
|
"workspace": {
|
||||||
|
"dependencies": [
|
||||||
|
"jsr:@openai/openai@^6.17.0",
|
||||||
|
"jsr:@std/path@^1.1.4"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
312
libro.ipynb
Normal file
312
libro.ipynb
Normal file
@@ -0,0 +1,312 @@
|
|||||||
|
{
|
||||||
|
"cells": [
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 8,
|
||||||
|
"id": "9f587bf1",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"import { load } from \"jsr:@std/dotenv\";\n",
|
||||||
|
"import OpenAI from \"jsr:@openai/openai\";\n",
|
||||||
|
"\n",
|
||||||
|
"const _ = await load({ export: true });\n",
|
||||||
|
"const openai = new OpenAI();"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 9,
|
||||||
|
"id": "4650126c",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"const safeName = (s: string) => s.replace(/[<>:\"/\\\\|?*\\x00-\\x1F]/g, \"_\").trim();\n",
|
||||||
|
"const bookName =\n",
|
||||||
|
" \"Nmap Network Scanning Official Nmap Project Guide to Network Discovery and Security Scanning\";\n"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 10,
|
||||||
|
"id": "ae701b32",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"const ordinals = [\n",
|
||||||
|
" \"first\",\n",
|
||||||
|
" \"second\",\n",
|
||||||
|
" \"third\",\n",
|
||||||
|
" \"fourth\",\n",
|
||||||
|
" \"fifth\",\n",
|
||||||
|
" \"sixth\",\n",
|
||||||
|
" \"seventh\",\n",
|
||||||
|
" \"eighth\",\n",
|
||||||
|
" \"ninth\",\n",
|
||||||
|
" \"tenth\",\n",
|
||||||
|
" \"eleventh\",\n",
|
||||||
|
" \"twelfth\",\n",
|
||||||
|
" \"thirteenth\",\n",
|
||||||
|
" \"fourteenth\",\n",
|
||||||
|
" \"fifteenth\",\n",
|
||||||
|
" \"sixteenth\",\n",
|
||||||
|
" \"seventeenth\",\n",
|
||||||
|
" \"eighteenth\",\n",
|
||||||
|
" \"nineteenth\",\n",
|
||||||
|
" \"twentieth\",\n",
|
||||||
|
"];\n"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"id": "8bee369d",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"[\n",
|
||||||
|
" { title: \"01\", level: 1, page: 27, endpage: 50 },\n",
|
||||||
|
" { title: \"02\", level: 1, page: 51, endpage: 71 },\n",
|
||||||
|
" { title: \"03\", level: 1, page: 72, endpage: 97 },\n",
|
||||||
|
" { title: \"04\", level: 1, page: 98, endpage: 119 },\n",
|
||||||
|
" { title: \"05\", level: 1, page: 120, endpage: 158 },\n",
|
||||||
|
" { title: \"06\", level: 1, page: 159, endpage: 168 },\n",
|
||||||
|
" { title: \"07\", level: 1, page: 169, endpage: 193 },\n",
|
||||||
|
" { title: \"08\", level: 1, page: 194, endpage: 227 },\n",
|
||||||
|
" { title: \"09\", level: 1, page: 228, endpage: 278 },\n",
|
||||||
|
" { title: \"10\", level: 1, page: 279, endpage: 315 },\n",
|
||||||
|
" { title: \"11\", level: 1, page: 316, endpage: 326 },\n",
|
||||||
|
" { title: \"12\", level: 1, page: 327, endpage: 356 },\n",
|
||||||
|
" { title: \"13\", level: 1, page: 357, endpage: 382 },\n",
|
||||||
|
" { title: \"14\", level: 1, page: 383, endpage: 391 },\n",
|
||||||
|
" { title: \"15\", level: 1, page: 392, endpage: 392 }\n",
|
||||||
|
"]\n",
|
||||||
|
"Processing chapter: 01\n",
|
||||||
|
"Processing chapter: 02\n",
|
||||||
|
"Processing chapter: 03\n",
|
||||||
|
"Processing chapter: 04\n",
|
||||||
|
"Processing chapter: 05\n",
|
||||||
|
"Processing chapter: 06\n",
|
||||||
|
"Processing chapter: 07\n",
|
||||||
|
"Processing chapter: 08\n",
|
||||||
|
"Processing chapter: 09\n",
|
||||||
|
"Processing chapter: 10\n",
|
||||||
|
"Processing chapter: 11\n",
|
||||||
|
"Processing chapter: 12\n",
|
||||||
|
"Processing chapter: 13\n",
|
||||||
|
"Processing chapter: 14\n",
|
||||||
|
"Processing chapter: 15\n",
|
||||||
|
"Finished processing chapter: 15\n",
|
||||||
|
"Finished processing chapter: 11\n",
|
||||||
|
"Finished processing chapter: 14\n",
|
||||||
|
"Finished processing chapter: 05\n",
|
||||||
|
"Finished processing chapter: 07\n",
|
||||||
|
"Finished processing chapter: 06\n",
|
||||||
|
"Finished processing chapter: 12\n",
|
||||||
|
"Finished processing chapter: 04\n",
|
||||||
|
"Finished processing chapter: 03\n",
|
||||||
|
"Finished processing chapter: 01\n",
|
||||||
|
"Finished processing chapter: 13\n",
|
||||||
|
"Finished processing chapter: 09\n",
|
||||||
|
"Finished processing chapter: 08\n",
|
||||||
|
"Finished processing chapter: 02\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"import { PDFDocument } from \"https://cdn.skypack.dev/pdf-lib@^1.11.1?dts\";\n",
|
||||||
|
"\n",
|
||||||
|
"const INPUT_PDF_PATH = `./books/${bookName}.pdf`;\n",
|
||||||
|
"\n",
|
||||||
|
"type Bookmark = { title: string; page: number; level: number; endpage: number };\n",
|
||||||
|
"const bookmarks = await new Deno.Command(\"pdftk\", {\n",
|
||||||
|
" args: [\n",
|
||||||
|
" INPUT_PDF_PATH,\n",
|
||||||
|
" \"dump_data\",\n",
|
||||||
|
" \"output\",\n",
|
||||||
|
" \"-\",\n",
|
||||||
|
" ],\n",
|
||||||
|
"}).output().then(\n",
|
||||||
|
" (res) => new TextDecoder().decode(res.stdout),\n",
|
||||||
|
").then((data) => {\n",
|
||||||
|
" const lines = data.split(\"\\n\");\n",
|
||||||
|
" let bookmarks = [];\n",
|
||||||
|
" let currentBookmark: Partial<Bookmark> | null = null;\n",
|
||||||
|
"\n",
|
||||||
|
" for (const line of lines) {\n",
|
||||||
|
" if (line.startsWith(\"BookmarkBegin\")) {\n",
|
||||||
|
" if (currentBookmark) {\n",
|
||||||
|
" bookmarks.push(currentBookmark as Bookmark);\n",
|
||||||
|
" }\n",
|
||||||
|
" currentBookmark = {};\n",
|
||||||
|
" } else if (line.startsWith(\"BookmarkTitle:\")) {\n",
|
||||||
|
" if (currentBookmark) {\n",
|
||||||
|
" currentBookmark.title = line.replace(\"BookmarkTitle: \", \"\").trim();\n",
|
||||||
|
" }\n",
|
||||||
|
" } else if (line.startsWith(\"BookmarkLevel:\")) {\n",
|
||||||
|
" if (currentBookmark) {\n",
|
||||||
|
" currentBookmark.level = parseInt(\n",
|
||||||
|
" line.replace(\"BookmarkLevel: \", \"\").trim(),\n",
|
||||||
|
" );\n",
|
||||||
|
" }\n",
|
||||||
|
" } else if (line.startsWith(\"BookmarkPageNumber:\")) {\n",
|
||||||
|
" if (currentBookmark) {\n",
|
||||||
|
" currentBookmark.page = parseInt(\n",
|
||||||
|
" line.replace(\"BookmarkPageNumber: \", \"\").trim(),\n",
|
||||||
|
" );\n",
|
||||||
|
" }\n",
|
||||||
|
" }\n",
|
||||||
|
" }\n",
|
||||||
|
" if (currentBookmark) {\n",
|
||||||
|
" bookmarks.push(currentBookmark as Bookmark);\n",
|
||||||
|
" }\n",
|
||||||
|
"\n",
|
||||||
|
" bookmarks = bookmarks.filter((b) => b.level === 1).slice(8);\n",
|
||||||
|
"\n",
|
||||||
|
" for (let i = 0; i < bookmarks.length; i++) {\n",
|
||||||
|
" const current = bookmarks[i] as Bookmark;\n",
|
||||||
|
" const next = bookmarks[i + 1] as Bookmark | undefined;\n",
|
||||||
|
" const currentPage = current.page ?? 0;\n",
|
||||||
|
" const nextPage = next?.page ?? 0;\n",
|
||||||
|
" current.endpage = nextPage\n",
|
||||||
|
" ? Math.max(currentPage, nextPage - 1)\n",
|
||||||
|
" : currentPage;\n",
|
||||||
|
" }\n",
|
||||||
|
"\n",
|
||||||
|
" return bookmarks;\n",
|
||||||
|
"});\n",
|
||||||
|
"console.log(bookmarks);\n",
|
||||||
|
"\n",
|
||||||
|
"const promises = [];\n",
|
||||||
|
"for (const [idx, ch] of bookmarks.entries()) {\n",
|
||||||
|
" async function processChapter(idx: number, title: string, file: File) {\n",
|
||||||
|
" const upload = await openai.files.create({\n",
|
||||||
|
" file,\n",
|
||||||
|
" purpose: \"user_data\",\n",
|
||||||
|
" });\n",
|
||||||
|
"\n",
|
||||||
|
" const response = await openai.responses.create({\n",
|
||||||
|
" model: \"gpt-5.2\",\n",
|
||||||
|
" reasoning: { effort: \"xhigh\" },\n",
|
||||||
|
" input: [{\n",
|
||||||
|
" role: \"user\",\n",
|
||||||
|
" content: [\n",
|
||||||
|
" {\n",
|
||||||
|
" type: \"input_text\",\n",
|
||||||
|
" text:\n",
|
||||||
|
" `A complete, exhaustive and very detailed mind map in markmap syntax of only and only this ${\n",
|
||||||
|
" ordinals[idx]\n",
|
||||||
|
" } chapter`,\n",
|
||||||
|
" },\n",
|
||||||
|
" { type: \"input_file\", file_id: upload.id },\n",
|
||||||
|
" ],\n",
|
||||||
|
" }],\n",
|
||||||
|
" metadata: {\n",
|
||||||
|
" chapter: title,\n",
|
||||||
|
" ordinal: ordinals[idx],\n",
|
||||||
|
" },\n",
|
||||||
|
" });\n",
|
||||||
|
" console.log(`Processing chapter: ${title} at ${response.id}`);\n",
|
||||||
|
"\n",
|
||||||
|
" await openai.files.delete(upload.id);\n",
|
||||||
|
"\n",
|
||||||
|
" const mindMapContent = response.output_text;\n",
|
||||||
|
" await Deno.writeTextFile(\n",
|
||||||
|
" `./mindmaps/${safeName(title)}.md`,\n",
|
||||||
|
" mindMapContent,\n",
|
||||||
|
" );\n",
|
||||||
|
"\n",
|
||||||
|
" console.log(`Finished processing chapter: ${title}`);\n",
|
||||||
|
" }\n",
|
||||||
|
" {\n",
|
||||||
|
" // Split the chapter into a separate PDF file\n",
|
||||||
|
" const pdfBytes = await Deno.readFile(INPUT_PDF_PATH);\n",
|
||||||
|
" const pdfDoc = await PDFDocument.load(pdfBytes);\n",
|
||||||
|
" const chapterPdf = await PDFDocument.create();\n",
|
||||||
|
"\n",
|
||||||
|
" const startPage = ch.page - 1; // zero-based index\n",
|
||||||
|
" const endPage = ch.endpage - 1; // zero-based index\n",
|
||||||
|
"\n",
|
||||||
|
" const pagesToCopy = await chapterPdf.copyPages(\n",
|
||||||
|
" pdfDoc,\n",
|
||||||
|
" Array.from(\n",
|
||||||
|
" { length: endPage - startPage + 1 },\n",
|
||||||
|
" (_, i) => i + startPage,\n",
|
||||||
|
" ),\n",
|
||||||
|
" );\n",
|
||||||
|
"\n",
|
||||||
|
" type PdfPage =\n",
|
||||||
|
" import(\"https://cdn.skypack.dev/pdf-lib@^1.11.1?dts\").PDFPage;\n",
|
||||||
|
"\n",
|
||||||
|
" pagesToCopy.forEach((page: PdfPage): void => {\n",
|
||||||
|
" chapterPdf.addPage(page);\n",
|
||||||
|
" });\n",
|
||||||
|
"\n",
|
||||||
|
" const chapterPdfBytes = await chapterPdf.save();\n",
|
||||||
|
" const chapterFile = new File(\n",
|
||||||
|
" [chapterPdfBytes],\n",
|
||||||
|
" `${safeName(ch.title)}.pdf`,\n",
|
||||||
|
" { type: \"application/pdf\" },\n",
|
||||||
|
" );\n",
|
||||||
|
" promises.push(processChapter(idx, ch.title, chapterFile));\n",
|
||||||
|
" }\n",
|
||||||
|
"}\n",
|
||||||
|
"await Promise.all(promises);\n"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 5,
|
||||||
|
"id": "7333f395",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"data": {
|
||||||
|
"text/plain": [
|
||||||
|
"{ object: \u001b[32m\"file\"\u001b[39m, deleted: \u001b[33mtrue\u001b[39m, id: \u001b[32m\"file-X2CJi3gozhJniBURG2qfsm\"\u001b[39m }"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"execution_count": 5,
|
||||||
|
"metadata": {},
|
||||||
|
"output_type": "execute_result"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"// delete all files from openai file storage\n",
|
||||||
|
"const userFiles = await openai.files.list({ purpose: \"user_data\" });\n",
|
||||||
|
"for (const file of userFiles.data) {\n",
|
||||||
|
" await openai.files.delete(file.id);\n",
|
||||||
|
"}\n"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"id": "b8a5dc2c",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": []
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"metadata": {
|
||||||
|
"kernelspec": {
|
||||||
|
"display_name": "Deno",
|
||||||
|
"language": "typescript",
|
||||||
|
"name": "deno"
|
||||||
|
},
|
||||||
|
"language_info": {
|
||||||
|
"codemirror_mode": "typescript",
|
||||||
|
"file_extension": ".ts",
|
||||||
|
"mimetype": "text/x.typescript",
|
||||||
|
"name": "typescript",
|
||||||
|
"nbconvert_exporter": "script",
|
||||||
|
"pygments_lexer": "typescript",
|
||||||
|
"version": "5.9.2"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"nbformat": 4,
|
||||||
|
"nbformat_minor": 5
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user