refactor(data augmentation): changed the tracking strategy of the data augmentation pipeline and its source code

data/interim/issue-report-classification/post-analysis/nasa-cfs/gemini-2.5-flash.csv.dvc

@@ -0,0 +1,5 @@
+outs:
+- md5: 810fc90fdb20cda56f1e20b589003f98
+  size: 20647734
+  hash: md5
+  path: gemini-2.5-flash.csv

data/interim/issue-report-classification/post-analysis/nlbse24/gemini-2.5-flash.csv.dvc

@@ -0,0 +1,5 @@
+outs:
+- md5: 3bbf67ae914c8c59277683c671224433
+  size: 15235640
+  hash: md5
+  path: gemini-2.5-flash.csv

data/interim/issue-report-classification/post-transform/nasa-cfs/gemini-2.5-flash.csv.dvc

@@ -0,0 +1,5 @@
+outs:
+- md5: 08fda2e3a09e5ef4f2f1386e7fd160d8
+  size: 7017011
+  hash: md5
+  path: gemini-2.5-flash.csv

data/interim/issue-report-classification/post-transform/nasa-cfs/gemini-2.5-flash/easy.csv.dvc

@@ -0,0 +1,5 @@
+outs:
+- md5: 22b83d918fe8304d12861b9b42d6fc70
+  size: 607787
+  hash: md5
+  path: easy.csv

data/interim/issue-report-classification/post-transform/nasa-cfs/gemini-2.5-flash/hard.csv.dvc

@@ -0,0 +1,5 @@
+outs:
+- md5: 910e201f4bb25c88bcff7dce1ae51f0b
+  size: 2311768
+  hash: md5
+  path: hard.csv

data/interim/issue-report-classification/post-transform/nasa-cfs/gemini-2.5-flash/medium.csv.dvc

@@ -0,0 +1,5 @@
+outs:
+- md5: dfd102bbce2ae8719ad33776272bf33e
+  size: 1621288
+  hash: md5
+  path: medium.csv

data/interim/issue-report-classification/post-transform/nlbse24/gemini-2.5-flash.csv.dvc

@@ -0,0 +1,5 @@
+outs:
+- md5: 4b65cf5eaa84c8868395727819e1501e
+  size: 8443153
+  hash: md5
+  path: gemini-2.5-flash.csv

data/interim/issue-report-classification/post-transform/nlbse24/gemini-2.5-flash/easy.csv.dvc

@@ -0,0 +1,5 @@
+outs:
+- md5: e62e5ae07e4f2611280006dd0dba7961
+  size: 586850
+  hash: md5
+  path: easy.csv

data/interim/issue-report-classification/post-transform/nlbse24/gemini-2.5-flash/hard.csv.dvc

@@ -0,0 +1,5 @@
+outs:
+- md5: 467982c597b82fd5df60400027b80161
+  size: 2460914
+  hash: md5
+  path: hard.csv

data/interim/issue-report-classification/post-transform/nlbse24/gemini-2.5-flash/medium.csv.dvc

@@ -0,0 +1,5 @@
+outs:
+- md5: 8aaeb6866bce768ee5d64042814e9016
+  size: 1978713
+  hash: md5
+  path: medium.csv

data/interim/issue-report-classification/soft-cleaned/nasa/cfs_test.csv.dvc

@@ -0,0 +1,5 @@
+outs:
+- md5: 1f761ca5d6c85c8726a234a542184970
+  size: 519291
+  hash: md5
+  path: cfs_test.csv

data/interim/issue-report-classification/soft-cleaned/nasa/cfs_train.csv.dvc

@@ -0,0 +1,5 @@
+outs:
+- md5: 39f4dae2567b2a2c4e9e0edf2534bad7
+  size: 1969469
+  hash: md5
+  path: cfs_train.csv

data/interim/issue-report-classification/soft-cleaned/nlbse24/issues_test.csv.dvc

@@ -0,0 +1,5 @@
+outs:
+- md5: c8b5bdc4006fbbdb555c3236c9a6d462
+  size: 3454159
+  hash: md5
+  path: issues_test.csv

data/interim/issue-report-classification/soft-cleaned/nlbse24/issues_train.csv.dvc

@@ -0,0 +1,5 @@
+outs:
+- md5: f21545a0f19ff9fad6568549a5a02181
+  size: 3686102
+  hash: md5
+  path: issues_train.csv

dvc.lock

@@ -1,434 +0,0 @@
-schema: '2.0'
-stages:
-  analyze_nlbse24:
-    cmd: python syntetic_issue_report_data_generation/augmentation/analyze.py 
-      --input-csv 
-      data/interim/issue-report-classification/soft-cleaned/nlbse24/issues_train.csv
-      --input-column issue --label-column label --context-file 
-      rsc/prompts/contexts/nlbse24.md --output-file 
-      data/interim/issue-report-classification/post-analysis/nlbse24/gemini-2.5-flash.csv
-      --provider google --model gemini-2.5-flash
-    deps:
-    - path: 
-        data/interim/issue-report-classification/soft-cleaned/nlbse24/issues_train.csv
-      hash: md5
-      md5: f21545a0f19ff9fad6568549a5a02181
-      size: 3686102
-    - path: rsc/prompts/analize.md
-      hash: md5
-      md5: ccacb11a2da7672e990f389e6359e8d4
-      size: 7832
-    - path: rsc/prompts/contexts/nlbse24.md
-      hash: md5
-      md5: 50984698bf834f05052caf60e91e04dd
-      size: 2454
-    - path: syntetic_issue_report_data_generation/augmentation/analyze.py
-      hash: md5
-      md5: 5f0f30547aaf9eb63a5acf2ccaff49c4
-      size: 10973
-    outs:
-    - path: 
-        data/interim/issue-report-classification/post-analysis/nlbse24/gemini-2.5-flash.csv
-      hash: md5
-      md5: 3bbf67ae914c8c59277683c671224433
-      size: 15235640
-  analyze_nasa_cfs:
-    cmd: python syntetic_issue_report_data_generation/augmentation/analyze.py 
-      --input-csv 
-      data/interim/issue-report-classification/soft-cleaned/nasa/cfs_train.csv 
-      --input-column issue --label-column label --context-file 
-      rsc/prompts/contexts/nasa-cfs.md --output-file 
-      data/interim/issue-report-classification/post-analysis/nasa-cfs/gemini-2.5-flash.csv
-      --provider google --model gemini-2.5-flash
-    deps:
-    - path: 
-        data/interim/issue-report-classification/soft-cleaned/nasa/cfs_train.csv
-      hash: md5
-      md5: 39f4dae2567b2a2c4e9e0edf2534bad7
-      size: 1969469
-    - path: rsc/prompts/analize.md
-      hash: md5
-      md5: ccacb11a2da7672e990f389e6359e8d4
-      size: 7832
-    - path: rsc/prompts/contexts/nasa-cfs.md
-      hash: md5
-      md5: 27fc32ef713da793f3d373b4c69b1f32
-      size: 5515
-    - path: syntetic_issue_report_data_generation/augmentation/analyze.py
-      hash: md5
-      md5: 5f0f30547aaf9eb63a5acf2ccaff49c4
-      size: 10973
-    outs:
-    - path: 
-        data/interim/issue-report-classification/post-analysis/nasa-cfs/gemini-2.5-flash.csv
-      hash: md5
-      md5: 810fc90fdb20cda56f1e20b589003f98
-      size: 20647734
-  transform_nlbse24:
-    cmd: python syntetic_issue_report_data_generation/augmentation/generate.py 
-      --input-csv 
-      data/interim/issue-report-classification/soft-cleaned/nlbse24/issues_train.csv
-      --analysis-csv 
-      data/interim/issue-report-classification/post-analysis/nlbse24/gemini-2.5-flash.csv
-      --input-column issue --label-column label --context-file 
-      rsc/prompts/contexts/nlbse24.md --method one_to_one_transform_all_3 
-      --output-file 
-      data/interim/issue-report-classification/post-transform/nlbse24/gemini-2.5-flash.csv
-      --provider google --model gemini-2.5-flash
-    deps:
-    - path: 
-        data/interim/issue-report-classification/post-analysis/nlbse24/gemini-2.5-flash.csv
-      hash: md5
-      md5: 3bbf67ae914c8c59277683c671224433
-      size: 15235640
-    - path: 
-        data/interim/issue-report-classification/soft-cleaned/nlbse24/issues_train.csv
-      hash: md5
-      md5: f21545a0f19ff9fad6568549a5a02181
-      size: 3686102
-    - path: rsc/prompts/contexts/nlbse24.md
-      hash: md5
-      md5: 50984698bf834f05052caf60e91e04dd
-      size: 2454
-    - path: rsc/prompts/transform/one-to-one-transform-all-3.md
-      hash: md5
-      md5: 12b1947ed4d4aead9942b9bbc7c49465
-      size: 2546
-    - path: syntetic_issue_report_data_generation/augmentation/analyze.py
-      hash: md5
-      md5: 5f0f30547aaf9eb63a5acf2ccaff49c4
-      size: 10973
-    - path: syntetic_issue_report_data_generation/augmentation/generate.py
-      hash: md5
-      md5: 06d0bae579db0fcb6aa2c35aec6f91c2
-      size: 7960
-    - path: syntetic_issue_report_data_generation/augmentation/utils.py
-      hash: md5
-      md5: b7205414ff7e3b58b05180e215b52ffa
-      size: 1982
-    outs:
-    - path: 
-        data/interim/issue-report-classification/post-transform/nlbse24/gemini-2.5-flash.csv
-      hash: md5
-      md5: 4b65cf5eaa84c8868395727819e1501e
-      size: 8443153
-  split_nlbse24_by_difficulty:
-    cmd: python 
-      syntetic_issue_report_data_generation/augmentation/split_by_difficulty.py 
-      --file 
-      data/interim/issue-report-classification/post-transform/nlbse24/gemini-2.5-flash.csv
-      --output-dir 
-      data/interim/issue-report-classification/post-transform/nlbse24/gemini-2.5-flash
-      --original-dataset 
-      ./data/interim/issue-report-classification/soft-cleaned/nlbse24/issues_train.csv
-    deps:
-    - path: 
-        data/interim/issue-report-classification/post-transform/nlbse24/gemini-2.5-flash.csv
-      hash: md5
-      md5: 4b65cf5eaa84c8868395727819e1501e
-      size: 8443153
-    - path: 
-        data/interim/issue-report-classification/soft-cleaned/nlbse24/issues_train.csv
-      hash: md5
-      md5: f21545a0f19ff9fad6568549a5a02181
-      size: 3686102
-    - path: 
-        syntetic_issue_report_data_generation/augmentation/split_by_difficulty.py
-      hash: md5
-      md5: e02f350ea800033d339b380ad2b4363b
-      size: 2299
-    outs:
-    - path: 
-        data/interim/issue-report-classification/post-transform/nlbse24/gemini-2.5-flash/easy.csv
-      hash: md5
-      md5: e62e5ae07e4f2611280006dd0dba7961
-      size: 586850
-    - path: 
-        data/interim/issue-report-classification/post-transform/nlbse24/gemini-2.5-flash/hard.csv
-      hash: md5
-      md5: 467982c597b82fd5df60400027b80161
-      size: 2460914
-    - path: 
-        data/interim/issue-report-classification/post-transform/nlbse24/gemini-2.5-flash/medium.csv
-      hash: md5
-      md5: 8aaeb6866bce768ee5d64042814e9016
-      size: 1978713
-  transform_nasa_cfs:
-    cmd: python syntetic_issue_report_data_generation/augmentation/generate.py 
-      --input-csv 
-      data/interim/issue-report-classification/soft-cleaned/nasa/cfs_train.csv 
-      --analysis-csv 
-      data/interim/issue-report-classification/post-analysis/nasa-cfs/gemini-2.5-flash.csv
-      --input-column issue --label-column label --context-file 
-      rsc/prompts/contexts/nasa-cfs.md --method one_to_one_transform_all_3 
-      --output-file 
-      data/interim/issue-report-classification/post-transform/nasa-cfs/gemini-2.5-flash.csv
-      --provider google --model gemini-2.5-flash
-    deps:
-    - path: 
-        data/interim/issue-report-classification/post-analysis/nasa-cfs/gemini-2.5-flash.csv
-      hash: md5
-      md5: 810fc90fdb20cda56f1e20b589003f98
-      size: 20647734
-    - path: 
-        data/interim/issue-report-classification/soft-cleaned/nasa/cfs_train.csv
-      hash: md5
-      md5: 39f4dae2567b2a2c4e9e0edf2534bad7
-      size: 1969469
-    - path: rsc/prompts/contexts/nasa-cfs.md
-      hash: md5
-      md5: 27fc32ef713da793f3d373b4c69b1f32
-      size: 5515
-    - path: rsc/prompts/transform/one-to-one-transform-all-3.md
-      hash: md5
-      md5: 12b1947ed4d4aead9942b9bbc7c49465
-      size: 2546
-    - path: syntetic_issue_report_data_generation/augmentation/analyze.py
-      hash: md5
-      md5: 5f0f30547aaf9eb63a5acf2ccaff49c4
-      size: 10973
-    - path: syntetic_issue_report_data_generation/augmentation/generate.py
-      hash: md5
-      md5: 06d0bae579db0fcb6aa2c35aec6f91c2
-      size: 7960
-    - path: syntetic_issue_report_data_generation/augmentation/utils.py
-      hash: md5
-      md5: b7205414ff7e3b58b05180e215b52ffa
-      size: 1982
-    outs:
-    - path: 
-        data/interim/issue-report-classification/post-transform/nasa-cfs/gemini-2.5-flash.csv
-      hash: md5
-      md5: 08fda2e3a09e5ef4f2f1386e7fd160d8
-      size: 7017011
-  preprocess_nlbse24:
-    cmd: python syntetic_issue_report_data_generation/augmentation/preprocess.py
-      --dataset nlbse24_train --output-dir 
-      data/interim/issue-report-classification/pre-analysis
-    deps:
-    - path: data/raw/issue-report-classification/nlbse24/issues_train.csv
-      hash: md5
-      md5: 1a1a99191592d38b3ec54a51d8d673c1
-      size: 3680400
-    - path: syntetic_issue_report_data_generation/augmentation/preprocess.py
-      hash: md5
-      md5: 0fc37014f5a08ef1c290898110a67c6e
-      size: 1574
-    outs:
-    - path: 
-        data/interim/issue-report-classification/pre-analysis/nlbse24_train.csv
-      hash: md5
-      md5: 1ca462f7b317236d8e540d953564ee73
-      size: 3630886
-  preprocess_nasa_cfs:
-    cmd: python syntetic_issue_report_data_generation/augmentation/preprocess.py
-      --dataset nasa_cfs_train --output-dir 
-      data/interim/issue-report-classification/pre-analysis
-    deps:
-    - path: data/raw/issue-report-classification/nasa/cfs_train.csv
-      hash: md5
-      md5: 01b5946e58649c0afa0613716c2259ef
-      size: 1961092
-    - path: syntetic_issue_report_data_generation/augmentation/preprocess.py
-      hash: md5
-      md5: 0fc37014f5a08ef1c290898110a67c6e
-      size: 1574
-    outs:
-    - path: 
-        data/interim/issue-report-classification/pre-analysis/nasa_cfs_train.csv
-      hash: md5
-      md5: 5094cc3998129fc469a6ca3aac967694
-      size: 1883320
-  split_nasa_cfs_by_difficulty:
-    cmd: python 
-      syntetic_issue_report_data_generation/augmentation/split_by_difficulty.py 
-      --file 
-      data/interim/issue-report-classification/post-transform/nasa-cfs/gemini-2.5-flash.csv
-      --output-dir 
-      data/interim/issue-report-classification/post-transform/nasa-cfs/gemini-2.5-flash
-      --original-dataset 
-      ./data/interim/issue-report-classification/soft-cleaned/nasa/cfs_train.csv
-    deps:
-    - path: 
-        data/interim/issue-report-classification/post-transform/nasa-cfs/gemini-2.5-flash.csv
-      hash: md5
-      md5: 08fda2e3a09e5ef4f2f1386e7fd160d8
-      size: 7017011
-    - path: 
-        data/interim/issue-report-classification/soft-cleaned/nasa/cfs_train.csv
-      hash: md5
-      md5: 39f4dae2567b2a2c4e9e0edf2534bad7
-      size: 1969469
-    - path: 
-        syntetic_issue_report_data_generation/augmentation/split_by_difficulty.py
-      hash: md5
-      md5: e02f350ea800033d339b380ad2b4363b
-      size: 2299
-    outs:
-    - path: 
-        data/interim/issue-report-classification/post-transform/nasa-cfs/gemini-2.5-flash/easy.csv
-      hash: md5
-      md5: 22b83d918fe8304d12861b9b42d6fc70
-      size: 607787
-    - path: 
-        data/interim/issue-report-classification/post-transform/nasa-cfs/gemini-2.5-flash/hard.csv
-      hash: md5
-      md5: 910e201f4bb25c88bcff7dce1ae51f0b
-      size: 2311768
-    - path: 
-        data/interim/issue-report-classification/post-transform/nasa-cfs/gemini-2.5-flash/medium.csv
-      hash: md5
-      md5: dfd102bbce2ae8719ad33776272bf33e
-      size: 1621288
-  soft_clean:
-    cmd: echo "hi"
-    deps:
-    - path: data/raw/issue-report-classification/nasa/cfs_test.csv
-      hash: md5
-      md5: 25fc494a76ff8b7e0bde317bc21aee11
-      size: 517219
-    - path: data/raw/issue-report-classification/nasa/cfs_train.csv
-      hash: md5
-      md5: 01b5946e58649c0afa0613716c2259ef
-      size: 1961092
-    - path: data/raw/issue-report-classification/nasa/fprime_test.csv
-      hash: md5
-      md5: 1765403a9cda44565be66f5099548285
-      size: 154541
-    - path: data/raw/issue-report-classification/nasa/fprime_train.csv
-      hash: md5
-      md5: e7dbfad07b298d9e69dd438a8d3cbff4
-      size: 638050
-    - path: data/raw/issue-report-classification/nasa/nasa_test_sample.csv
-      hash: md5
-      md5: c9574f535c664cce4c8e58b8745b3fa6
-      size: 7844
-    - path: data/raw/issue-report-classification/nasa/nasa_train_sample.csv
-      hash: md5
-      md5: 2c3199c3c33df94c3ef8363d04ae4d97
-      size: 15962
-    - path: data/raw/issue-report-classification/nlbse24/issues_test.csv
-      hash: md5
-      md5: 44e171e338c727cbc93bb8240e88aa1e
-      size: 3449390
-    - path: data/raw/issue-report-classification/nlbse24/issues_train.csv
-      hash: md5
-      md5: 1a1a99191592d38b3ec54a51d8d673c1
-      size: 3680400
-    outs:
-    - path: 
-        data/interim/issue-report-classification/soft-cleaned/nasa/cfs_test.csv
-      hash: md5
-      md5: 1d7c921b78807befb300f877e5ccad23
-      size: 517765
-    - path: 
-        data/interim/issue-report-classification/soft-cleaned/nasa/cfs_train.csv
-      hash: md5
-      md5: 919adb09810a708c708007585ccc5763
-      size: 1963260
-    - path: 
-        data/interim/issue-report-classification/soft-cleaned/nasa/fprime_test.csv
-      hash: md5
-      md5: 8786472104c9aa23b60381cdc9f97861
-      size: 154611
-    - path: 
-        data/interim/issue-report-classification/soft-cleaned/nasa/fprime_train.csv
-      hash: md5
-      md5: a4852e1657d3e84d602eacefe233833e
-      size: 638585
-    - path: 
-        data/interim/issue-report-classification/soft-cleaned/nasa/nasa_test_sample.csv
-      hash: md5
-      md5: e8a9304e5632243977a9ccb38df22fee
-      size: 7855
-    - path: 
-        data/interim/issue-report-classification/soft-cleaned/nasa/nasa_train_sample.csv
-      hash: md5
-      md5: 5bccd9369e16384130509c166cd3cbdf
-      size: 15973
-    - path: 
-        data/interim/issue-report-classification/soft-cleaned/nlbse24/issues_test.csv
-      hash: md5
-      md5: 5083327fc87e50cc1ef926c01527ed07
-      size: 3449960
-    - path: 
-        data/interim/issue-report-classification/soft-cleaned/nlbse24/issues_train.csv
-      hash: md5
-      md5: 6124c859150e47b0940346880f55a2f6
-      size: 3681901
-  preprocess_nlbse24_train:
-    cmd: python syntetic_issue_report_data_generation/preprocess.py --dataset 
-      nlbse24_train --output 
-      data/interim/issue-report-classification/soft-cleaned/nlbse24/issues_train.csv
-    deps:
-    - path: data/raw/issue-report-classification/nlbse24/issues_train.csv
-      hash: md5
-      md5: 1a1a99191592d38b3ec54a51d8d673c1
-      size: 3680400
-    - path: syntetic_issue_report_data_generation/preprocess.py
-      hash: md5
-      md5: ea154c395ecb7bd6f18f0fdf99b8e532
-      size: 4973
-    outs:
-    - path: 
-        data/interim/issue-report-classification/soft-cleaned/nlbse24/issues_train.csv
-      hash: md5
-      md5: f21545a0f19ff9fad6568549a5a02181
-      size: 3686102
-  preprocess_nlbse24_test:
-    cmd: python syntetic_issue_report_data_generation/preprocess.py --dataset 
-      nlbse24_test --output 
-      data/interim/issue-report-classification/soft-cleaned/nlbse24/issues_test.csv
-    deps:
-    - path: data/raw/issue-report-classification/nlbse24/issues_test.csv
-      hash: md5
-      md5: 44e171e338c727cbc93bb8240e88aa1e
-      size: 3449390
-    - path: syntetic_issue_report_data_generation/preprocess.py
-      hash: md5
-      md5: ea154c395ecb7bd6f18f0fdf99b8e532
-      size: 4973
-    outs:
-    - path: 
-        data/interim/issue-report-classification/soft-cleaned/nlbse24/issues_test.csv
-      hash: md5
-      md5: c8b5bdc4006fbbdb555c3236c9a6d462
-      size: 3454159
-  preprocess_nasa_cfs_train:
-    cmd: python syntetic_issue_report_data_generation/preprocess.py --dataset 
-      nasa_cfs_train --output 
-      data/interim/issue-report-classification/soft-cleaned/nasa/cfs_train.csv
-    deps:
-    - path: data/raw/issue-report-classification/nasa/cfs_train.csv
-      hash: md5
-      md5: 01b5946e58649c0afa0613716c2259ef
-      size: 1961092
-    - path: syntetic_issue_report_data_generation/preprocess.py
-      hash: md5
-      md5: ea154c395ecb7bd6f18f0fdf99b8e532
-      size: 4973
-    outs:
-    - path: 
-        data/interim/issue-report-classification/soft-cleaned/nasa/cfs_train.csv
-      hash: md5
-      md5: 39f4dae2567b2a2c4e9e0edf2534bad7
-      size: 1969469
-  preprocess_nasa_cfs_test:
-    cmd: python syntetic_issue_report_data_generation/preprocess.py --dataset 
-      nasa_cfs_test --output 
-      data/interim/issue-report-classification/soft-cleaned/nasa/cfs_test.csv
-    deps:
-    - path: data/raw/issue-report-classification/nasa/cfs_test.csv
-      hash: md5
-      md5: 25fc494a76ff8b7e0bde317bc21aee11
-      size: 517219
-    - path: syntetic_issue_report_data_generation/preprocess.py
-      hash: md5
-      md5: ea154c395ecb7bd6f18f0fdf99b8e532
-      size: 4973
-    outs:
-    - path: 
-        data/interim/issue-report-classification/soft-cleaned/nasa/cfs_test.csv
-      hash: md5
-      md5: 1f761ca5d6c85c8726a234a542184970
-      size: 519291

dvc.yaml

@@ -1,177 +0,0 @@
-stages:
-
-  # ==================== PREPROCESSING STAGES ====================
-
-  preprocess_nlbse24_train:
-    cmd: >-
-      python syntetic_issue_report_data_generation/preprocess.py
-      --dataset nlbse24_train
-      --output data/interim/issue-report-classification/soft-cleaned/nlbse24/issues_train.csv
-    deps:
-      - data/raw/issue-report-classification/nlbse24/issues_train.csv
-      - syntetic_issue_report_data_generation/preprocess.py
-    outs:
-      - data/interim/issue-report-classification/soft-cleaned/nlbse24/issues_train.csv
-
-  preprocess_nlbse24_test:
-    cmd: >-
-      python syntetic_issue_report_data_generation/preprocess.py
-      --dataset nlbse24_test
-      --output data/interim/issue-report-classification/soft-cleaned/nlbse24/issues_test.csv
-    deps:
-      - data/raw/issue-report-classification/nlbse24/issues_test.csv
-      - syntetic_issue_report_data_generation/preprocess.py
-    outs:
-      - data/interim/issue-report-classification/soft-cleaned/nlbse24/issues_test.csv
-
-  preprocess_nasa_cfs_train:
-    cmd: >-
-      python syntetic_issue_report_data_generation/preprocess.py
-      --dataset nasa_cfs_train
-      --output data/interim/issue-report-classification/soft-cleaned/nasa/cfs_train.csv
-    deps:
-      - data/raw/issue-report-classification/nasa/cfs_train.csv
-      - syntetic_issue_report_data_generation/preprocess.py
-    outs:
-      - data/interim/issue-report-classification/soft-cleaned/nasa/cfs_train.csv
-
-  preprocess_nasa_cfs_test:
-    cmd: >-
-      python syntetic_issue_report_data_generation/preprocess.py
-      --dataset nasa_cfs_test
-      --output data/interim/issue-report-classification/soft-cleaned/nasa/cfs_test.csv
-    deps:
-      - data/raw/issue-report-classification/nasa/cfs_test.csv
-      - syntetic_issue_report_data_generation/preprocess.py
-    outs:
-      - data/interim/issue-report-classification/soft-cleaned/nasa/cfs_test.csv
-
-  # ==================== ANALYSIS STAGES ====================
-  
-  analyze_nlbse24:
-    cmd: >-
-      python syntetic_issue_report_data_generation/augmentation/analyze.py
-      --input-csv data/interim/issue-report-classification/soft-cleaned/nlbse24/issues_train.csv
-      --input-column issue
-      --label-column label
-      --context-file rsc/prompts/contexts/nlbse24.md
-      --output-file data/interim/issue-report-classification/post-analysis/nlbse24/gemini-2.5-flash.csv
-      --provider google
-      --model gemini-2.5-flash
-    deps:
-      - data/interim/issue-report-classification/soft-cleaned/nlbse24/issues_train.csv
-      - rsc/prompts/contexts/nlbse24.md
-      - rsc/prompts/analize.md
-      - syntetic_issue_report_data_generation/augmentation/analyze.py
-      # - syntetic_issue_report_data_generation/augmentation/utils.py
-    outs:
-      - data/interim/issue-report-classification/post-analysis/nlbse24/gemini-2.5-flash.csv:
-          persist: true
-
-  analyze_nasa_cfs:
-    cmd: >-
-      python syntetic_issue_report_data_generation/augmentation/analyze.py
-      --input-csv data/interim/issue-report-classification/soft-cleaned/nasa/cfs_train.csv
-      --input-column issue
-      --label-column label
-      --context-file rsc/prompts/contexts/nasa-cfs.md
-      --output-file data/interim/issue-report-classification/post-analysis/nasa-cfs/gemini-2.5-flash.csv
-      --provider google
-      --model gemini-2.5-flash
-    deps:
-      - data/interim/issue-report-classification/soft-cleaned/nasa/cfs_train.csv
-      - rsc/prompts/contexts/nasa-cfs.md
-      - rsc/prompts/analize.md
-      - syntetic_issue_report_data_generation/augmentation/analyze.py
-      # - syntetic_issue_report_data_generation/augmentation/utils.py
-    outs:
-      - data/interim/issue-report-classification/post-analysis/nasa-cfs/gemini-2.5-flash.csv:
-          persist: true
-
-  # ==================== TRANSFORM STAGES ====================
-
-  transform_nlbse24:
-    cmd: >-
-      python syntetic_issue_report_data_generation/augmentation/generate.py
-      --input-csv data/interim/issue-report-classification/soft-cleaned/nlbse24/issues_train.csv
-      --analysis-csv data/interim/issue-report-classification/post-analysis/nlbse24/gemini-2.5-flash.csv
-      --input-column issue
-      --label-column label
-      --context-file rsc/prompts/contexts/nlbse24.md
-      --method one_to_one_transform_all_3
-      --output-file data/interim/issue-report-classification/post-transform/nlbse24/gemini-2.5-flash.csv
-      --provider google
-      --model gemini-2.5-flash
-    deps:
-      - data/interim/issue-report-classification/soft-cleaned/nlbse24/issues_train.csv
-      - data/interim/issue-report-classification/post-analysis/nlbse24/gemini-2.5-flash.csv
-      - rsc/prompts/contexts/nlbse24.md
-      - rsc/prompts/transform/one-to-one-transform-all-3.md
-      - syntetic_issue_report_data_generation/augmentation/generate.py
-      # - syntetic_issue_report_data_generation/augmentation/analyze.py
-      # - syntetic_issue_report_data_generation/augmentation/utils.py
-    outs:
-      - data/interim/issue-report-classification/post-transform/nlbse24/gemini-2.5-flash.csv:
-          persist: true
-
-  transform_nasa_cfs:
-    cmd: >-
-      python syntetic_issue_report_data_generation/augmentation/generate.py
-      --input-csv data/interim/issue-report-classification/soft-cleaned/nasa/cfs_train.csv
-      --analysis-csv data/interim/issue-report-classification/post-analysis/nasa-cfs/gemini-2.5-flash.csv
-      --input-column issue
-      --label-column label
-      --context-file rsc/prompts/contexts/nasa-cfs.md
-      --method one_to_one_transform_all_3
-      --output-file data/interim/issue-report-classification/post-transform/nasa-cfs/gemini-2.5-flash.csv
-      --provider google
-      --model gemini-2.5-flash
-    deps:
-      - data/interim/issue-report-classification/soft-cleaned/nasa/cfs_train.csv
-      - data/interim/issue-report-classification/post-analysis/nasa-cfs/gemini-2.5-flash.csv
-      - rsc/prompts/contexts/nasa-cfs.md
-      - rsc/prompts/transform/one-to-one-transform-all-3.md
-      - syntetic_issue_report_data_generation/augmentation/generate.py
-      # - syntetic_issue_report_data_generation/augmentation/analyze.py
-      # - syntetic_issue_report_data_generation/augmentation/utils.py
-    outs:
-      - data/interim/issue-report-classification/post-transform/nasa-cfs/gemini-2.5-flash.csv:
-          persist: true
-
-  # ==================== SPLIT BY DIFFICULTY STAGES ====================
-  
-  split_nlbse24_by_difficulty:
-    cmd: >-
-      python syntetic_issue_report_data_generation/augmentation/split_by_difficulty.py
-      --file data/interim/issue-report-classification/post-transform/nlbse24/gemini-2.5-flash.csv
-      --output-dir data/interim/issue-report-classification/post-transform/nlbse24/gemini-2.5-flash
-      --original-dataset ./data/interim/issue-report-classification/soft-cleaned/nlbse24/issues_train.csv
-    deps:
-      - data/interim/issue-report-classification/post-transform/nlbse24/gemini-2.5-flash.csv
-      - syntetic_issue_report_data_generation/augmentation/split_by_difficulty.py
-      - data/interim/issue-report-classification/soft-cleaned/nlbse24/issues_train.csv
-    outs:
-      - data/interim/issue-report-classification/post-transform/nlbse24/gemini-2.5-flash/easy.csv:
-          persist: true
-      - data/interim/issue-report-classification/post-transform/nlbse24/gemini-2.5-flash/medium.csv:
-          persist: true
-      - data/interim/issue-report-classification/post-transform/nlbse24/gemini-2.5-flash/hard.csv:
-          persist: true
-    
-  split_nasa_cfs_by_difficulty:
-    cmd: >-
-      python syntetic_issue_report_data_generation/augmentation/split_by_difficulty.py
-      --file data/interim/issue-report-classification/post-transform/nasa-cfs/gemini-2.5-flash.csv
-      --output-dir data/interim/issue-report-classification/post-transform/nasa-cfs/gemini-2.5-flash
-      --original-dataset ./data/interim/issue-report-classification/soft-cleaned/nasa/cfs_train.csv
-    deps:
-      - data/interim/issue-report-classification/post-transform/nasa-cfs/gemini-2.5-flash.csv
-      - syntetic_issue_report_data_generation/augmentation/split_by_difficulty.py
-      - data/interim/issue-report-classification/soft-cleaned/nasa/cfs_train.csv
-    outs:
-      - data/interim/issue-report-classification/post-transform/nasa-cfs/gemini-2.5-flash/easy.csv:
-          persist: true
-      - data/interim/issue-report-classification/post-transform/nasa-cfs/gemini-2.5-flash/medium.csv:
-          persist: true
-      - data/interim/issue-report-classification/post-transform/nasa-cfs/gemini-2.5-flash/hard.csv:
-          persist: true

pipeline.ps1

@@ -0,0 +1,259 @@
+# DVC Pipeline Script for Capibara
+# This script runs the entire data processing pipeline
+
+param(
+    [string]$Stage = "all",
+    [switch]$Help
+)
+
+if ($Help) {
+    Write-Host @"
+Capibara Pipeline Script
+========================
+
+Usage: .\pipeline.ps1 [-Stage <stage_name>] [-Help]
+
+Stages:
+  all                           - Run all stages (default)
+  preprocess                    - Run all preprocessing stages
+  preprocess_nlbse24_train      - Preprocess NLBSE24 training data
+  preprocess_nlbse24_test       - Preprocess NLBSE24 test data
+  preprocess_nasa_cfs_train     - Preprocess NASA CFS training data
+  preprocess_nasa_cfs_test      - Preprocess NASA CFS test data
+  analyze                       - Run all analysis stages
+  analyze_nlbse24               - Analyze NLBSE24 data
+  analyze_nasa_cfs              - Analyze NASA CFS data
+  transform                     - Run all transform stages
+  transform_nlbse24             - Transform NLBSE24 data
+  transform_nasa_cfs            - Transform NASA CFS data
+  split                         - Run all split by difficulty stages
+  split_nlbse24_by_difficulty   - Split NLBSE24 by difficulty
+  split_nasa_cfs_by_difficulty  - Split NASA CFS by difficulty
+
+Examples:
+  .\pipeline.ps1                        # Run all stages
+  .\pipeline.ps1 -Stage preprocess      # Run only preprocessing
+  .\pipeline.ps1 -Stage analyze_nlbse24 # Run specific stage
+"@
+    exit 0
+}
+
+# Set error action preference
+$ErrorActionPreference = "Stop"
+
+# Define colors for output
+function Write-StageHeader {
+    param([string]$Message)
+    Write-Host ""
+    Write-Host "==================== $Message ====================" -ForegroundColor Cyan
+    Write-Host ""
+}
+
+function Write-StageInfo {
+    param([string]$Message)
+    Write-Host "[INFO] $Message" -ForegroundColor Green
+}
+
+function Write-StageError {
+    param([string]$Message)
+    Write-Host "[ERROR] $Message" -ForegroundColor Red
+}
+
+# ==================== PREPROCESSING STAGES ====================
+
+function Invoke-PreprocessNlbse24Train {
+    Write-StageInfo "Running: preprocess_nlbse24_train"
+    python syntetic_issue_report_data_generation/preprocess.py `
+        --dataset nlbse24_train `
+        --output data/interim/issue-report-classification/soft-cleaned/nlbse24/issues_train.csv
+    if ($LASTEXITCODE -ne 0) { throw "preprocess_nlbse24_train failed" }
+}
+
+function Invoke-PreprocessNlbse24Test {
+    Write-StageInfo "Running: preprocess_nlbse24_test"
+    python syntetic_issue_report_data_generation/preprocess.py `
+        --dataset nlbse24_test `
+        --output data/interim/issue-report-classification/soft-cleaned/nlbse24/issues_test.csv
+    if ($LASTEXITCODE -ne 0) { throw "preprocess_nlbse24_test failed" }
+}
+
+function Invoke-PreprocessNasaCfsTrain {
+    Write-StageInfo "Running: preprocess_nasa_cfs_train"
+    python syntetic_issue_report_data_generation/preprocess.py `
+        --dataset nasa_cfs_train `
+        --output data/interim/issue-report-classification/soft-cleaned/nasa/cfs_train.csv
+    if ($LASTEXITCODE -ne 0) { throw "preprocess_nasa_cfs_train failed" }
+}
+
+function Invoke-PreprocessNasaCfsTest {
+    Write-StageInfo "Running: preprocess_nasa_cfs_test"
+    python syntetic_issue_report_data_generation/preprocess.py `
+        --dataset nasa_cfs_test `
+        --output data/interim/issue-report-classification/soft-cleaned/nasa/cfs_test.csv
+    if ($LASTEXITCODE -ne 0) { throw "preprocess_nasa_cfs_test failed" }
+}
+
+function Invoke-AllPreprocess {
+    Write-StageHeader "PREPROCESSING STAGES"
+    Invoke-PreprocessNlbse24Train
+    Invoke-PreprocessNlbse24Test
+    Invoke-PreprocessNasaCfsTrain
+    Invoke-PreprocessNasaCfsTest
+}
+
+# ==================== ANALYSIS STAGES ====================
+
+function Invoke-AnalyzeNlbse24 {
+    Write-StageInfo "Running: analyze_nlbse24"
+    python syntetic_issue_report_data_generation/augmentation/analyze.py `
+        --input-csv data/interim/issue-report-classification/soft-cleaned/nlbse24/issues_train.csv `
+        --input-column issue `
+        --label-column label `
+        --context-file rsc/prompts/contexts/nlbse24.md `
+        --output-file data/interim/issue-report-classification/post-analysis/nlbse24/gemini-2.5-flash.csv `
+        --provider google `
+        --model gemini-2.5-flash
+    if ($LASTEXITCODE -ne 0) { throw "analyze_nlbse24 failed" }
+}
+
+function Invoke-AnalyzeNasaCfs {
+    Write-StageInfo "Running: analyze_nasa_cfs"
+    python syntetic_issue_report_data_generation/augmentation/analyze.py `
+        --input-csv data/interim/issue-report-classification/soft-cleaned/nasa/cfs_train.csv `
+        --input-column issue `
+        --label-column label `
+        --context-file rsc/prompts/contexts/nasa-cfs.md `
+        --output-file data/interim/issue-report-classification/post-analysis/nasa-cfs/gemini-2.5-flash.csv `
+        --provider google `
+        --model gemini-2.5-flash
+    if ($LASTEXITCODE -ne 0) { throw "analyze_nasa_cfs failed" }
+}
+
+function Invoke-AllAnalyze {
+    Write-StageHeader "ANALYSIS STAGES"
+    Invoke-AnalyzeNlbse24
+    Invoke-AnalyzeNasaCfs
+}
+
+# ==================== TRANSFORM STAGES ====================
+
+function Invoke-TransformNlbse24 {
+    Write-StageInfo "Running: transform_nlbse24"
+    python syntetic_issue_report_data_generation/augmentation/generate.py `
+        --input-csv data/interim/issue-report-classification/soft-cleaned/nlbse24/issues_train.csv `
+        --analysis-csv data/interim/issue-report-classification/post-analysis/nlbse24/gemini-2.5-flash.csv `
+        --input-column issue `
+        --label-column label `
+        --context-file rsc/prompts/contexts/nlbse24.md `
+        --method one_to_one_transform_all_3 `
+        --output-file data/interim/issue-report-classification/post-transform/nlbse24/gemini-2.5-flash.csv `
+        --provider google `
+        --model gemini-2.5-flash
+    if ($LASTEXITCODE -ne 0) { throw "transform_nlbse24 failed" }
+}
+
+function Invoke-TransformNasaCfs {
+    Write-StageInfo "Running: transform_nasa_cfs"
+    python syntetic_issue_report_data_generation/augmentation/generate.py `
+        --input-csv data/interim/issue-report-classification/soft-cleaned/nasa/cfs_train.csv `
+        --analysis-csv data/interim/issue-report-classification/post-analysis/nasa-cfs/gemini-2.5-flash.csv `
+        --input-column issue `
+        --label-column label `
+        --context-file rsc/prompts/contexts/nasa-cfs.md `
+        --method one_to_one_transform_all_3 `
+        --output-file data/interim/issue-report-classification/post-transform/nasa-cfs/gemini-2.5-flash.csv `
+        --provider google `
+        --model gemini-2.5-flash
+    if ($LASTEXITCODE -ne 0) { throw "transform_nasa_cfs failed" }
+}
+
+function Invoke-AllTransform {
+    Write-StageHeader "TRANSFORM STAGES"
+    Invoke-TransformNlbse24
+    Invoke-TransformNasaCfs
+}
+
+# ==================== SPLIT BY DIFFICULTY STAGES ====================
+
+function Invoke-SplitNlbse24ByDifficulty {
+    Write-StageInfo "Running: split_nlbse24_by_difficulty"
+    python syntetic_issue_report_data_generation/augmentation/split_by_difficulty.py `
+        --file data/interim/issue-report-classification/post-transform/nlbse24/gemini-2.5-flash.csv `
+        --output-dir data/interim/issue-report-classification/post-transform/nlbse24/gemini-2.5-flash `
+        --original-dataset ./data/interim/issue-report-classification/soft-cleaned/nlbse24/issues_train.csv
+    if ($LASTEXITCODE -ne 0) { throw "split_nlbse24_by_difficulty failed" }
+}
+
+function Invoke-SplitNasaCfsByDifficulty {
+    Write-StageInfo "Running: split_nasa_cfs_by_difficulty"
+    python syntetic_issue_report_data_generation/augmentation/split_by_difficulty.py `
+        --file data/interim/issue-report-classification/post-transform/nasa-cfs/gemini-2.5-flash.csv `
+        --output-dir data/interim/issue-report-classification/post-transform/nasa-cfs/gemini-2.5-flash `
+        --original-dataset ./data/interim/issue-report-classification/soft-cleaned/nasa/cfs_train.csv
+    if ($LASTEXITCODE -ne 0) { throw "split_nasa_cfs_by_difficulty failed" }
+}
+
+function Invoke-AllSplit {
+    Write-StageHeader "SPLIT BY DIFFICULTY STAGES"
+    Invoke-SplitNlbse24ByDifficulty
+    Invoke-SplitNasaCfsByDifficulty
+}
+
+# ==================== MAIN EXECUTION ====================
+
+function Invoke-AllStages {
+    Invoke-AllPreprocess
+    Invoke-AllAnalyze
+    Invoke-AllTransform
+    Invoke-AllSplit
+}
+
+# Main execution block
+try {
+    Write-Host "Capibara Pipeline" -ForegroundColor Yellow
+    Write-Host "=================" -ForegroundColor Yellow
+    Write-Host "Stage: $Stage" -ForegroundColor Yellow
+    
+    switch ($Stage.ToLower()) {
+        "all" { Invoke-AllStages }
+        
+        # Preprocessing stages
+        "preprocess" { Invoke-AllPreprocess }
+        "preprocess_nlbse24_train" { Invoke-PreprocessNlbse24Train }
+        "preprocess_nlbse24_test" { Invoke-PreprocessNlbse24Test }
+        "preprocess_nasa_cfs_train" { Invoke-PreprocessNasaCfsTrain }
+        "preprocess_nasa_cfs_test" { Invoke-PreprocessNasaCfsTest }
+        
+        # Analysis stages
+        "analyze" { Invoke-AllAnalyze }
+        "analyze_nlbse24" { Invoke-AnalyzeNlbse24 }
+        "analyze_nasa_cfs" { Invoke-AnalyzeNasaCfs }
+        
+        # Transform stages
+        "transform" { Invoke-AllTransform }
+        "transform_nlbse24" { Invoke-TransformNlbse24 }
+        "transform_nasa_cfs" { Invoke-TransformNasaCfs }
+        
+        # Split by difficulty stages
+        "split" { Invoke-AllSplit }
+        "split_nlbse24_by_difficulty" { Invoke-SplitNlbse24ByDifficulty }
+        "split_nasa_cfs_by_difficulty" { Invoke-SplitNasaCfsByDifficulty }
+        
+        default {
+            Write-StageError "Unknown stage: $Stage"
+            Write-Host "Use -Help to see available stages"
+            exit 1
+        }
+    }
+    
+    Write-Host ""
+    Write-Host "==================== PIPELINE COMPLETED ====================" -ForegroundColor Green
+    Write-Host ""
+    
+} catch {
+    Write-StageError $_.Exception.Message
+    Write-Host ""
+    Write-Host "==================== PIPELINE FAILED ====================" -ForegroundColor Red
+    Write-Host ""
+    exit 1
+}

syntetic_issue_report_data_generation/config.py

@@ -11,7 +11,7 @@ PROJ_ROOT = Path(__file__).resolve().parents[1]
 DATA_DIR = PROJ_ROOT / "data"
 RAW_DATA_DIR = DATA_DIR / "raw"
 INTERIM_DATA_DIR = DATA_DIR / "interim"
-SOFT_CLEANED_DATA_DIR = DATA_DIR / "soft_cleaned"
+SOFT_CLEANED_DATA_DIR = INTERIM_DATA_DIR / "soft-cleaned"
 PROCESSED_DATA_DIR = DATA_DIR / "processed"
 EXTERNAL_DATA_DIR = DATA_DIR / "external"