zyratlo commented on code in PR #8767: URL: https://github.com/apache/texera/pull/8767#discussion_r4149469142
########## frontend/src/app/workspace/service/notebook-migration/folder-assembly.ts: ########## @@ -0,0 +1,355 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { ScriptSegmentation, splitScriptLines } from "./script-segmentation"; + +/** + * Assembles a folder of Python sources into the one numbered document the migration LLM sees. + * + * Converted as a single input, so the model reports line ranges as it does for a script and + * nothing downstream learns folders exist; a banner line before each file carries provenance. + * Lines split through the segmenter's own splitScriptLines, so assembly, numbering and slicing + * agree on what a line is. + */ + +// Never project code, matched per segment so a cache or vendored dependency is caught at any +// depth. Hidden segments are covered separately by the leading-dot rule. +const EXCLUDED_SEGMENTS = new Set(["__pycache__", "venv", "site-packages", "node_modules"]); + +// Build output at the root, but ordinary package names deeper in a tree (src/build/ can be real +// code). Dropping real code silently is worse than letting a build tree hit the caps. +const EXCLUDED_ROOT_DIRECTORIES = new Set(["build", "dist"]); + +// Non-Python files are named in the layout, never read. Capped because a folder can hold +// thousands of data files and the layout is orientation, not an inventory. +export const MAX_LISTED_OTHER_FILES = 20; + +/** + * Caps on one conversion, refused before any request is sent. Sized by the reply, not the prompt: + * the conversion asks for every line of code back and sets no output budget, so the model's output + * ceiling binds long before its context does. Raise once a larger folder has been measured. + */ +export const MAX_FOLDER_FILES = 100; +export const MAX_FOLDER_CHARACTERS = 60_000; + +// Pre-read guard for the character cap: UTF-8 never uses fewer bytes than characters, so a +// selection this far over cannot come in under it, and one huge file would otherwise hang the tab +// before the character cap, which needs the files read, could run. +export const MAX_FOLDER_BYTES = MAX_FOLDER_CHARACTERS * 4; + +export const FILE_BANNER_PREFIX = "# ===== FILE: "; +export const FILE_BANNER_SUFFIX = " ====="; + +/** The line that introduces a file in the assembled document. */ +export function fileBanner(path: string): string { + return `${FILE_BANNER_PREFIX}${path}${FILE_BANNER_SUFFIX}`; +} + +export interface FolderFile { + // Path relative to the selected folder, "/" separated, as webkitRelativePath reports it. + path: string; + source: string; +} + +// 1-indexed and inclusive bounds within the assembled document. +export interface LineSpan { + startLine: number; + endLine: number; +} + +// Spans the banner line through the file's last line. +export interface FolderFileSpan extends LineSpan { + path: string; +} + +export interface FolderDocument { + // The concatenated source, which is what gets numbered for the prompt and segmented afterwards. + source: string; + // Each file's banner line, passed to segmentScript so no cell straddles a file. + forcedBoundaries: number[]; + // The files that made it into the document, in the order they appear in it. + files: FolderFileSpan[]; + /** + * An indented listing of the folder, sent ahead of the document as prompt text. Never part of + * `source`: numbering it would shift every line the model reports, and the segmenter would emit + * it as a cell of directory listing. Also the only place non-Python files are named. + */ + tree: string; +} + +export interface FolderDocumentOptions { + // The selected folder's own name, used as the layout's root label. + rootName?: string; + // Paths in the selection that hold no Python source. Listed in the layout, never read. + otherPaths?: readonly string[]; +} + +/** Either shape an upload control hands a picked file over in: the File itself, or a wrapper. */ +export interface PickedFile { + webkitRelativePath?: string; + originFileObj?: { webkitRelativePath?: string }; +} + +/** + * The path a directory picker reported for a picked file. Both shapes must be read: ng-zorro's + * beforeUpload gets the File itself with a uid attached and only wraps it in `originFileObj` + * later, so checking one alone silently yields no path. + */ +export function pickedFilePath(file: PickedFile | undefined): string { + return file?.originFileObj?.webkitRelativePath ?? file?.webkitRelativePath ?? ""; +} + +/** The browser File behind a picked file, whichever of the two shapes the control used. */ +export function pickedFileObject(file: PickedFile): File { + return (file.originFileObj ?? file) as unknown as File; Review Comment: Fixed in 65ca25808, with a narrower guard than suggested: `instanceof Blob` and throw, no duck-typing fallback. The fallback branch would let an object with the right members through while FileReader still rejects it, and `typeof Blob === "undefined"` cannot happen where FileReader exists. The throw surfaces in generateFromFolder's catch, which shows the message to the user, so it is worded for that. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
