Copilot commented on code in PR #8767: URL: https://github.com/apache/texera/pull/8767#discussion_r4149246693
########## frontend/src/app/workspace/service/notebook-migration/folder-assembly.ts: ########## @@ -0,0 +1,355 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { ScriptSegmentation, splitScriptLines } from "./script-segmentation"; + +/** + * Assembles a folder of Python sources into the one numbered document the migration LLM sees. + * + * Converted as a single input, so the model reports line ranges as it does for a script and + * nothing downstream learns folders exist; a banner line before each file carries provenance. + * Lines split through the segmenter's own splitScriptLines, so assembly, numbering and slicing + * agree on what a line is. + */ + +// Never project code, matched per segment so a cache or vendored dependency is caught at any +// depth. Hidden segments are covered separately by the leading-dot rule. +const EXCLUDED_SEGMENTS = new Set(["__pycache__", "venv", "site-packages", "node_modules"]); + +// Build output at the root, but ordinary package names deeper in a tree (src/build/ can be real +// code). Dropping real code silently is worse than letting a build tree hit the caps. +const EXCLUDED_ROOT_DIRECTORIES = new Set(["build", "dist"]); + +// Non-Python files are named in the layout, never read. Capped because a folder can hold +// thousands of data files and the layout is orientation, not an inventory. +export const MAX_LISTED_OTHER_FILES = 20; + +/** + * Caps on one conversion, refused before any request is sent. Sized by the reply, not the prompt: + * the conversion asks for every line of code back and sets no output budget, so the model's output + * ceiling binds long before its context does. Raise once a larger folder has been measured. + */ +export const MAX_FOLDER_FILES = 100; +export const MAX_FOLDER_CHARACTERS = 60_000; + +// Pre-read guard for the character cap: UTF-8 never uses fewer bytes than characters, so a +// selection this far over cannot come in under it, and one huge file would otherwise hang the tab +// before the character cap, which needs the files read, could run. +export const MAX_FOLDER_BYTES = MAX_FOLDER_CHARACTERS * 4; + +export const FILE_BANNER_PREFIX = "# ===== FILE: "; +export const FILE_BANNER_SUFFIX = " ====="; + +/** The line that introduces a file in the assembled document. */ +export function fileBanner(path: string): string { + return `${FILE_BANNER_PREFIX}${path}${FILE_BANNER_SUFFIX}`; +} + +export interface FolderFile { + // Path relative to the selected folder, "/" separated, as webkitRelativePath reports it. + path: string; + source: string; +} + +// 1-indexed and inclusive bounds within the assembled document. +export interface LineSpan { + startLine: number; + endLine: number; +} + +// Spans the banner line through the file's last line. +export interface FolderFileSpan extends LineSpan { + path: string; +} + +export interface FolderDocument { + // The concatenated source, which is what gets numbered for the prompt and segmented afterwards. + source: string; + // Each file's banner line, passed to segmentScript so no cell straddles a file. + forcedBoundaries: number[]; + // The files that made it into the document, in the order they appear in it. + files: FolderFileSpan[]; + /** + * An indented listing of the folder, sent ahead of the document as prompt text. Never part of + * `source`: numbering it would shift every line the model reports, and the segmenter would emit + * it as a cell of directory listing. Also the only place non-Python files are named. + */ + tree: string; +} + +export interface FolderDocumentOptions { + // The selected folder's own name, used as the layout's root label. + rootName?: string; + // Paths in the selection that hold no Python source. Listed in the layout, never read. + otherPaths?: readonly string[]; +} + +/** Either shape an upload control hands a picked file over in: the File itself, or a wrapper. */ +export interface PickedFile { + webkitRelativePath?: string; + originFileObj?: { webkitRelativePath?: string }; +} + +/** + * The path a directory picker reported for a picked file. Both shapes must be read: ng-zorro's + * beforeUpload gets the File itself with a uid attached and only wraps it in `originFileObj` + * later, so checking one alone silently yields no path. + */ +export function pickedFilePath(file: PickedFile | undefined): string { + return file?.originFileObj?.webkitRelativePath ?? file?.webkitRelativePath ?? ""; +} + +/** The browser File behind a picked file, whichever of the two shapes the control used. */ +export function pickedFileObject(file: PickedFile): File { + return (file.originFileObj ?? file) as unknown as File; Review Comment: `pickedFileObject()` can return a non-File object when `originFileObj` is absent (e.g., an `NzUploadFile`-like wrapper with just `uid`/`name`). That value will later be passed to `FileReader.readAsText(...)`, which will fail at runtime. Add a runtime guard (e.g., verify the candidate is a `File`/`Blob`-like object) and throw a clear error (or return null) so callers can surface a user-friendly message instead of crashing mid-read. ########## frontend/src/app/workspace/service/notebook-migration/migration-prompts.ts: ########## @@ -586,3 +586,235 @@ Here is an example of a mapping generated between the given example Python code } Now create a mapping for the UDFs and the original code you were given. For each UDF, report the line ranges of the original script whose logic that UDF implements. The code in those lines should be equivalent to what the UDF does. Lines that no UDF implements, such as imports or the data loading that the workflow's source operator replaces, can be left out entirely. Give the first line before the last within each range, and do not shift the numbers: they must match the prefixes you were shown. There could be any number of ranges and UDFs, so only create the correct number in the mapping. Only give the mapping. `; + +export const EXAMPLE_OF_MULTIPLE_UDF_CONVERSION_FOLDER = ` +Here is an example of breaking up a folder of Python files into multiple Texera UDFs. Format your response structure exactly like the given example. The "code" key contains a dictionary of the UDF IDs with their respective code. The "edges" key contains a list of pairs that contains the connections between UDFs. The "outputs" key contains a dictionary of the UDF IDs with a list of the output column names of the DataFrame that the UDF yields. The "entry_point" key names the one file that calls the pipeline's steps in order. The UDFs can branch and merge, it does not have to be a linear chain depending on your implementation. + +The folder's files are shown as one document, concatenated in the order given. A line of the form '# ===== FILE: <path> =====' marks where a file begins, and each line is prefixed by its line number and a '|'. Line numbers run continuously across the whole document and do not restart at each file. Both the banners and the number prefixes are annotations so that line ranges can be referred to later. They are not part of the code and must never appear in the code you generate. + +Note that main.py is the entry point: it is the file that actually runs, calling the functions the other two files define. The definitions it calls are copied into the UDFs, and the imports on lines 30 and 31 appear in no generated UDF. Note also that the layout names requirements.txt, which is not Python source and so does not appear in the document. + +Folder layout: +diabetes_analysis/ + data_prep.py + main.py + models.py + requirements.txt + +Original Code: +\`\`\`python + 1| # ===== FILE: data_prep.py ===== + 2| import pandas as pd + 3| import matplotlib.pyplot as plt + 4| + 5| + 6| def load_data(file_path): + 7| return pd.read_csv(file_path) + 8| + 9| +10| def clean(data): +11| data = data.drop_duplicates() +12| data = data.dropna() +13| return data +14| +15| +16| def summarize(data): +17| print("Minimum values:", data.min()) +18| print("Maximum values:", data.max()) +19| print("Mean values:", data.mean()) +20| return data +21| +22| +23| def boxplot_pregnancies(data): +24| plt.figure(figsize=(8, 6)) +25| plt.boxplot(data['Pregnancies'], vert=False, patch_artist=True) +26| plt.title('Boxplot of Pregnancies') +27| plt.xlabel('Number of Pregnancies') +28| plt.show() +29| # ===== FILE: main.py ===== +30| from data_prep import boxplot_pregnancies, clean, load_data, summarize +31| from models import split_and_scale, train_random_forest, train_svm +32| +33| data = load_data('diabetes.csv') +34| +35| data = clean(data) +36| summarize(data) +37| +38| boxplot_pregnancies(data) +39| +40| X_train, X_test, y_train, y_test = split_and_scale(data) +41| +42| train_random_forest(X_train, y_train, X_test, y_test) +43| train_svm(X_train, y_train, X_test, y_test) +44| # ===== FILE: models.py ===== +45| from sklearn.model_selection import train_test_split +46| from sklearn.ensemble import RandomForestClassifier +47| from sklearn.svm import SVC +48| from sklearn.metrics import accuracy_score +49| from sklearn.preprocessing import StandardScaler +50| +51| +52| def split_and_scale(data): +53| X = data.drop('Outcome', axis=1) +54| y = data['Outcome'] +55| X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42) +56| scaler = StandardScaler() +57| X_train = scaler.fit_transform(X_train) +58| X_test = scaler.transform(X_test) +59| return X_train, X_test, y_train, y_test +60| +61| +62| def train_random_forest(X_train, y_train, X_test, y_test): +63| rf_model = RandomForestClassifier(random_state=42) +64| rf_model.fit(X_train, y_train) +65| rf_pred = rf_model.predict(X_test) +66| rf_accuracy = accuracy_score(y_test, rf_pred) +67| print(f"Random Forest Accuracy: {rf_accuracy:.2%}") +68| +69| +70| def train_svm(X_train, y_train, X_test, y_test): +71| svm_model = SVC(random_state=42) +72| svm_model.fit(X_train, y_train) +73| svm_pred = svm_model.predict(X_test) +74| svm_accuracy = accuracy_score(y_test, svm_pred) +75| print(f"SVM Accuracy: {svm_accuracy:.2%}") +\`\`\` + +Texera UDF conversion: +\`\`\`json +{ + "entry_point": "main.py", + "code": { + "UDF1": "# UDF1\nfrom pytexera import *\nimport pandas as pd\nfrom typing import Iterator, Optional\n\nclass ProcessTableOperator(UDFTableOperator):\n\n @overrides\n def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]:\n # Remove duplicate rows\n data = table.drop_duplicates()\n\n # Remove rows with null values\n data = data.dropna()\n\n # Calculate statistics\n min_values = data.min()\n max_values = data.max()\n mean_values = data.mean()\n\n # Create a DataFrame to yield\n result_table = pd.DataFrame({\n 'min_values': [min_values],\n 'max_values': [max_values],\n 'mean_values': [mean_values],\n 'data': [data]\n })\n\n yield Table(result_table)", + "UDF2": "# UDF2\nfrom pytexera import *\nimport pandas as pd\nimport plotly.express as px\nimport plotly.io\nfrom typing import Iterator, Optional\n\nclass ProcessTableOperator(UDFTableOperator):\n def render_error(self, error_msg):\n return '''<h1>Boxplot is not available.</h1>\n <p>Reason is: {} </p>\n '''.format(error_msg)\n\n @overrides\n def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]:\n data = table['data'].iloc[0]\n\n if data.empty:\n yield {'html-content': self.render_error('input table is empty.')}\n return\n\n # Create a boxplot for the 'Pregnancies' field\n fig = px.box(data, x='Pregnancies')\n fig.update_layout(margin=dict(l=0, r=0, t=0, b=0))\n\n # Convert fig to HTML content\n html = plotly.io.to_html(fig, include_plotlyjs='cdn', auto_play=False)\n yield {'html-content': html}", + "UDF3": "# UDF3\nfrom pytexera import *\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler\nfrom typing import Iterator, Optional\n\nclass ProcessTableOperator(UDFTableOperator):\n\n @overrides\n def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]:\n data = table['data'].iloc[0]\n\n # Separate features and target variable\n X = data.drop('Outcome', axis=1)\n y = data['Outcome']\n\n # Split data into training and testing sets (80% train, 20% test)\n X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n\n scaler = StandardScaler()\n X_train = scaler.fit_transform(X_train)\n X_test = scaler.transform(X_test)\n\n # Create a DataFrame to yield\n result_table = pd.DataFrame({\n 'X_train': [X_train], 'X_test': [X_test],\n 'y_train': [y_tra in], 'y_test': [y_test]\n })\n\n yield Table(result_table)", + "UDF4": "# UDF4\nfrom pytexera import *\nimport pandas as pd\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.metrics import accuracy_score\nfrom typing import Iterator, Optional\n\nclass ProcessTableOperator(UDFTableOperator):\n\n @overrides\n def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]:\n X_train = table['X_train'].iloc[0]\n y_train = table['y_train'].iloc[0]\n X_test = table['X_test'].iloc[0]\n y_test = table['y_test'].iloc[0]\n\n # Train Random Forest model\n rf_model = RandomForestClassifier(random_state=42)\n rf_model.fit(X_train, y_train)\n rf_pred = rf_model.predict(X_test)\n rf_accuracy = accuracy_score(y_test, rf_pred)\n\n # Create a DataFrame to yield\n result_table = pd.DataFrame({\n 'rf_model': [rf_model],\n 'rf_accuracy': [rf_accuracy],\n 'X_test': [X_test],\n 'y_test': [y_test]\n })\n\n yield Table(result_table)", + "UDF5": "# UDF5\nfrom pytexera import *\nimport pandas as pd\nfrom sklearn.svm import SVC\nfrom sklearn.metrics import accuracy_score\nfrom typing import Iterator, Optional\n\nclass ProcessTableOperator(UDFTableOperator):\n\n @overrides\n def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]:\n X_train = table['X_train'].iloc[0]\n y_train = table['y_train'].iloc[0]\n X_test = table['X_test'].iloc[0]\n y_test = table['y_test'].iloc[0]\n\n # Train SVM model\n svm_model = SVC(random_state=42)\n svm_model.fit(X_train, y_train)\n svm_pred = svm_model.predict(X_test)\n svm_accuracy = accuracy_score(y_test, svm_pred)\n\n # Create a DataFrame to yield\n result_table = pd.DataFrame({\n 'svm_model': [svm_model],\n 'svm_accuracy': [svm_accuracy],\n 'X_test': [X_test],\n 'y_test': [y_test]\n })\n\n yield Table(result_tab le)" + }, + "edges": [ + ["UDF1", "UDF2"], + ["UDF1", "UDF3"], + ["UDF3", "UDF4"], + ["UDF3", "UDF5"] + ], + "outputs": { + "UDF1": ["min_values", "max_values", "mean_values", "data"], + "UDF2": ["html-content"], + "UDF3": ["X_train", "X_test", "y_train", "y_test"], + "UDF4": ["rf_model", "rf_accuracy", "X_test", "y_test"], + "UDF5": ["svm_model", "svm_accuracy", "X_test", "y_test"] + } +} +\`\`\``; + +export const FOLDER_WORKFLOW_PROMPT = `You are an expert in Python coding and workflow systems. +Many users of Texera system are non-technical, but the Python projects they provide are written by technical people. +They want to convert a folder of Python files into a single Texera workflow. +Your goal is to help convert these folders into a Texera workflow that non-technical users can use directly. +So do not remove or modify any classes or functions, preserve their names and structure as they are. +Ensure that all essential logic remains intact. +Create multiple Texera UDF codes using the provided Python code. +Number each UDF, starting at 1 and incrementing, by starting with a comment that states that UDF number. + +Use the class and function names as shown in ProcessTupleOperator, ProcessTableOperator, and ProcessBatchOperator. +Do not change the class names, function names, or input parameters. +Use the ones that make sense and split the code meaningfully as instructed. + +Use the starter code provided for Python UDFs. + +Use the documentation of Table, Tuple, or Batch to work with parameters within Texera UDF. +Do not import other libraries to define these types. + +There is no need for an __init__ function. Assume all inputs are valid pandas DataFrames, +so do not use .to_pandas(), .to_dataframe(), etc. Do not load data from a file in the first UDF; +the workflow's source operator supplies the initial data, so assume it is already given to you in the +table parameter. Replacing file-loading code with this input is the one exception to preserving all +original code (see below). +Ensure proper data flow between functions. Separate operators as if they will run in different files. + +Current UDF operators can only have one output. Build a dataframe to yield all necessary variables +and data. Ensure proper data flow for each UDF and all information is yielded (including training +and testing data) if subsequent UDFs need them. + +Ensure all necessary imports are included in each UDF code block. + +Each UDF operator should be in its own Python code block. Do not combine them into a single block. +Ensure import statements cover all used functions and separate them as necessary. Review Comment: The folder workflow prompt gives conflicting formatting instructions: it asks for “Python code block(s)” but also requires “raw JSON only” and “no markdown code fences.” This ambiguity can degrade model compliance (e.g., returning fenced code blocks or mixing formatting). Consider rewording the “code block” guidance to reference the JSON shape explicitly (e.g., “each UDF must be a separate string value under the `code` object; do not concatenate multiple UDF classes into one string”), and remove “code block” terminology that implies markdown fences. ########## frontend/src/app/workspace/service/notebook-migration/notebook-migration.service.ts: ########## @@ -128,17 +150,45 @@ export class NotebookMigrationService { public async sendScriptToAIGenerateWorkflow( scriptSource: string, modelType: string - ): Promise<{ workflowContent: WorkflowContent; mappingContent: MappingContent; notebook: Notebook }> { + ): Promise<GeneratedWorkflowContent> { + return this.generateFromSource(modelType, "script", migrationLLM => + migrationLLM.convertScriptToWorkflow(scriptSource) + ); + } + + /** + * Convert a folder of Python files, already assembled by parseFolder, into a workflow. + * + * Identical to the script path from the caller's side: the folder is one document by the time + * it gets here, so the derived notebook and the mapping have the same shape and are stored the + * same way. + */ + public async sendFolderToAIGenerateWorkflow( + folder: FolderDocument, + modelType: string + ): Promise<GeneratedWorkflowContent> { + return this.generateFromSource(modelType, "folder", migrationLLM => migrationLLM.convertFolderToWorkflow(folder)); + } + + /** + * Run one conversion of an input that arrives without cells, then unpack it into the shape + * callers store. The script and folder paths differ only in which conversion they ask for. + */ + private async generateFromSource( + modelType: string, + input: string, Review Comment: `input` is only used as a label in an error message and is currently typed as `string`, even though the implementation passes a small fixed set of values ("script", "folder"). Tightening this to a string-literal union (e.g., `"script" | "folder"`) makes the API self-documenting and prevents accidental typos from silently making logs inconsistent. ########## frontend/src/app/workspace/service/notebook-migration/migration-prompts.ts: ########## @@ -586,3 +586,235 @@ Here is an example of a mapping generated between the given example Python code } Now create a mapping for the UDFs and the original code you were given. For each UDF, report the line ranges of the original script whose logic that UDF implements. The code in those lines should be equivalent to what the UDF does. Lines that no UDF implements, such as imports or the data loading that the workflow's source operator replaces, can be left out entirely. Give the first line before the last within each range, and do not shift the numbers: they must match the prefixes you were shown. There could be any number of ranges and UDFs, so only create the correct number in the mapping. Only give the mapping. `; + +export const EXAMPLE_OF_MULTIPLE_UDF_CONVERSION_FOLDER = ` +Here is an example of breaking up a folder of Python files into multiple Texera UDFs. Format your response structure exactly like the given example. The "code" key contains a dictionary of the UDF IDs with their respective code. The "edges" key contains a list of pairs that contains the connections between UDFs. The "outputs" key contains a dictionary of the UDF IDs with a list of the output column names of the DataFrame that the UDF yields. The "entry_point" key names the one file that calls the pipeline's steps in order. The UDFs can branch and merge, it does not have to be a linear chain depending on your implementation. + +The folder's files are shown as one document, concatenated in the order given. A line of the form '# ===== FILE: <path> =====' marks where a file begins, and each line is prefixed by its line number and a '|'. Line numbers run continuously across the whole document and do not restart at each file. Both the banners and the number prefixes are annotations so that line ranges can be referred to later. They are not part of the code and must never appear in the code you generate. + +Note that main.py is the entry point: it is the file that actually runs, calling the functions the other two files define. The definitions it calls are copied into the UDFs, and the imports on lines 30 and 31 appear in no generated UDF. Note also that the layout names requirements.txt, which is not Python source and so does not appear in the document. + +Folder layout: +diabetes_analysis/ + data_prep.py + main.py + models.py + requirements.txt + +Original Code: +\`\`\`python + 1| # ===== FILE: data_prep.py ===== + 2| import pandas as pd + 3| import matplotlib.pyplot as plt + 4| + 5| + 6| def load_data(file_path): + 7| return pd.read_csv(file_path) + 8| + 9| +10| def clean(data): +11| data = data.drop_duplicates() +12| data = data.dropna() +13| return data +14| +15| +16| def summarize(data): +17| print("Minimum values:", data.min()) +18| print("Maximum values:", data.max()) +19| print("Mean values:", data.mean()) +20| return data +21| +22| +23| def boxplot_pregnancies(data): +24| plt.figure(figsize=(8, 6)) +25| plt.boxplot(data['Pregnancies'], vert=False, patch_artist=True) +26| plt.title('Boxplot of Pregnancies') +27| plt.xlabel('Number of Pregnancies') +28| plt.show() +29| # ===== FILE: main.py ===== +30| from data_prep import boxplot_pregnancies, clean, load_data, summarize +31| from models import split_and_scale, train_random_forest, train_svm +32| +33| data = load_data('diabetes.csv') +34| +35| data = clean(data) +36| summarize(data) +37| +38| boxplot_pregnancies(data) +39| +40| X_train, X_test, y_train, y_test = split_and_scale(data) +41| +42| train_random_forest(X_train, y_train, X_test, y_test) +43| train_svm(X_train, y_train, X_test, y_test) +44| # ===== FILE: models.py ===== +45| from sklearn.model_selection import train_test_split +46| from sklearn.ensemble import RandomForestClassifier +47| from sklearn.svm import SVC +48| from sklearn.metrics import accuracy_score +49| from sklearn.preprocessing import StandardScaler +50| +51| +52| def split_and_scale(data): +53| X = data.drop('Outcome', axis=1) +54| y = data['Outcome'] +55| X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42) +56| scaler = StandardScaler() +57| X_train = scaler.fit_transform(X_train) +58| X_test = scaler.transform(X_test) +59| return X_train, X_test, y_train, y_test +60| +61| +62| def train_random_forest(X_train, y_train, X_test, y_test): +63| rf_model = RandomForestClassifier(random_state=42) +64| rf_model.fit(X_train, y_train) +65| rf_pred = rf_model.predict(X_test) +66| rf_accuracy = accuracy_score(y_test, rf_pred) +67| print(f"Random Forest Accuracy: {rf_accuracy:.2%}") +68| +69| +70| def train_svm(X_train, y_train, X_test, y_test): +71| svm_model = SVC(random_state=42) +72| svm_model.fit(X_train, y_train) +73| svm_pred = svm_model.predict(X_test) +74| svm_accuracy = accuracy_score(y_test, svm_pred) +75| print(f"SVM Accuracy: {svm_accuracy:.2%}") +\`\`\` + +Texera UDF conversion: +\`\`\`json +{ + "entry_point": "main.py", + "code": { + "UDF1": "# UDF1\nfrom pytexera import *\nimport pandas as pd\nfrom typing import Iterator, Optional\n\nclass ProcessTableOperator(UDFTableOperator):\n\n @overrides\n def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]:\n # Remove duplicate rows\n data = table.drop_duplicates()\n\n # Remove rows with null values\n data = data.dropna()\n\n # Calculate statistics\n min_values = data.min()\n max_values = data.max()\n mean_values = data.mean()\n\n # Create a DataFrame to yield\n result_table = pd.DataFrame({\n 'min_values': [min_values],\n 'max_values': [max_values],\n 'mean_values': [mean_values],\n 'data': [data]\n })\n\n yield Table(result_table)", + "UDF2": "# UDF2\nfrom pytexera import *\nimport pandas as pd\nimport plotly.express as px\nimport plotly.io\nfrom typing import Iterator, Optional\n\nclass ProcessTableOperator(UDFTableOperator):\n def render_error(self, error_msg):\n return '''<h1>Boxplot is not available.</h1>\n <p>Reason is: {} </p>\n '''.format(error_msg)\n\n @overrides\n def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]:\n data = table['data'].iloc[0]\n\n if data.empty:\n yield {'html-content': self.render_error('input table is empty.')}\n return\n\n # Create a boxplot for the 'Pregnancies' field\n fig = px.box(data, x='Pregnancies')\n fig.update_layout(margin=dict(l=0, r=0, t=0, b=0))\n\n # Convert fig to HTML content\n html = plotly.io.to_html(fig, include_plotlyjs='cdn', auto_play=False)\n yield {'html-content': html}", + "UDF3": "# UDF3\nfrom pytexera import *\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler\nfrom typing import Iterator, Optional\n\nclass ProcessTableOperator(UDFTableOperator):\n\n @overrides\n def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]:\n data = table['data'].iloc[0]\n\n # Separate features and target variable\n X = data.drop('Outcome', axis=1)\n y = data['Outcome']\n\n # Split data into training and testing sets (80% train, 20% test)\n X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n\n scaler = StandardScaler()\n X_train = scaler.fit_transform(X_train)\n X_test = scaler.transform(X_test)\n\n # Create a DataFrame to yield\n result_table = pd.DataFrame({\n 'X_train': [X_train], 'X_test': [X_test],\n 'y_train': [y_tra in], 'y_test': [y_test]\n })\n\n yield Table(result_table)", + "UDF4": "# UDF4\nfrom pytexera import *\nimport pandas as pd\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.metrics import accuracy_score\nfrom typing import Iterator, Optional\n\nclass ProcessTableOperator(UDFTableOperator):\n\n @overrides\n def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]:\n X_train = table['X_train'].iloc[0]\n y_train = table['y_train'].iloc[0]\n X_test = table['X_test'].iloc[0]\n y_test = table['y_test'].iloc[0]\n\n # Train Random Forest model\n rf_model = RandomForestClassifier(random_state=42)\n rf_model.fit(X_train, y_train)\n rf_pred = rf_model.predict(X_test)\n rf_accuracy = accuracy_score(y_test, rf_pred)\n\n # Create a DataFrame to yield\n result_table = pd.DataFrame({\n 'rf_model': [rf_model],\n 'rf_accuracy': [rf_accuracy],\n 'X_test': [X_test],\n 'y_test': [y_test]\n })\n\n yield Table(result_table)", + "UDF5": "# UDF5\nfrom pytexera import *\nimport pandas as pd\nfrom sklearn.svm import SVC\nfrom sklearn.metrics import accuracy_score\nfrom typing import Iterator, Optional\n\nclass ProcessTableOperator(UDFTableOperator):\n\n @overrides\n def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]:\n X_train = table['X_train'].iloc[0]\n y_train = table['y_train'].iloc[0]\n X_test = table['X_test'].iloc[0]\n y_test = table['y_test'].iloc[0]\n\n # Train SVM model\n svm_model = SVC(random_state=42)\n svm_model.fit(X_train, y_train)\n svm_pred = svm_model.predict(X_test)\n svm_accuracy = accuracy_score(y_test, svm_pred)\n\n # Create a DataFrame to yield\n result_table = pd.DataFrame({\n 'svm_model': [svm_model],\n 'svm_accuracy': [svm_accuracy],\n 'X_test': [X_test],\n 'y_test': [y_test]\n })\n\n yield Table(result_tab le)" + }, + "edges": [ + ["UDF1", "UDF2"], + ["UDF1", "UDF3"], + ["UDF3", "UDF4"], + ["UDF3", "UDF5"] + ], + "outputs": { + "UDF1": ["min_values", "max_values", "mean_values", "data"], + "UDF2": ["html-content"], + "UDF3": ["X_train", "X_test", "y_train", "y_test"], + "UDF4": ["rf_model", "rf_accuracy", "X_test", "y_test"], + "UDF5": ["svm_model", "svm_accuracy", "X_test", "y_test"] + } +} +\`\`\``; + +export const FOLDER_WORKFLOW_PROMPT = `You are an expert in Python coding and workflow systems. +Many users of Texera system are non-technical, but the Python projects they provide are written by technical people. +They want to convert a folder of Python files into a single Texera workflow. +Your goal is to help convert these folders into a Texera workflow that non-technical users can use directly. +So do not remove or modify any classes or functions, preserve their names and structure as they are. +Ensure that all essential logic remains intact. +Create multiple Texera UDF codes using the provided Python code. +Number each UDF, starting at 1 and incrementing, by starting with a comment that states that UDF number. + +Use the class and function names as shown in ProcessTupleOperator, ProcessTableOperator, and ProcessBatchOperator. +Do not change the class names, function names, or input parameters. +Use the ones that make sense and split the code meaningfully as instructed. + +Use the starter code provided for Python UDFs. + +Use the documentation of Table, Tuple, or Batch to work with parameters within Texera UDF. +Do not import other libraries to define these types. + +There is no need for an __init__ function. Assume all inputs are valid pandas DataFrames, +so do not use .to_pandas(), .to_dataframe(), etc. Do not load data from a file in the first UDF; +the workflow's source operator supplies the initial data, so assume it is already given to you in the +table parameter. Replacing file-loading code with this input is the one exception to preserving all +original code (see below). +Ensure proper data flow between functions. Separate operators as if they will run in different files. + +Current UDF operators can only have one output. Build a dataframe to yield all necessary variables +and data. Ensure proper data flow for each UDF and all information is yielded (including training +and testing data) if subsequent UDFs need them. + +Ensure all necessary imports are included in each UDF code block. + +Each UDF operator should be in its own Python code block. Do not combine them into a single block. +Ensure import statements cover all used functions and separate them as necessary. + +It is VERY important that all of the original code in the folder's Python files is represented in the generated workflow. +Make sure that nothing in the original is removed and that the semantic meaning of what the original code was doing is retained. +The only exception is data-loading code (e.g. pd.read_csv); it is represented by the workflow's input/source operator rather than copied into a UDF. +If there are user-defined Python classes, include the entire class definition in the appropriate UDF(s) that use that class. +Always include the full class definition inside every UDF that references that class, including every UDF that constructs an object of it. +Python classes are allowed in Texera UDFs and follow the same semantics as standard Python. +They can be defined outside of ProcessTableOperator, ProcessTupleOperator, and ProcessBatchOperator. + +Return only the JSON formatted response, do not give any explanation. +Do not wrap the JSON in markdown code fences. Output raw JSON only. Review Comment: The folder workflow prompt gives conflicting formatting instructions: it asks for “Python code block(s)” but also requires “raw JSON only” and “no markdown code fences.” This ambiguity can degrade model compliance (e.g., returning fenced code blocks or mixing formatting). Consider rewording the “code block” guidance to reference the JSON shape explicitly (e.g., “each UDF must be a separate string value under the `code` object; do not concatenate multiple UDF classes into one string”), and remove “code block” terminology that implies markdown fences. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
