Update preprocessing scripts

Author: theQuert

nb/extract_netku.ipynb

@@ -1573,25 +1573,41 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 52,
+   "execution_count": 11,
    "id": "bb62fa4e-e331-4f0f-ae0a-fb5eece8ee71",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "168"
+      ]
+     },
+     "execution_count": 11,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
-    "hyps = csv_file.ref.to_list()"
+    "# hyps = csv_file.ref.to_list()\n",
+    "hyps = []\n",
+    "with open(\"/Users/quert/Documents/GitHub/edit_NetKu/dataset/same_secs_insert_labeled/final_hyp.src\", \"r\") as f:\n",
+    "    for line in f.readlines():\n",
+    "        hyps.append(line.strip())\n",
+    "len(hyps)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 53,
+   "execution_count": 12,
    "id": "7871b42c-1dbc-4bcb-b104-23bb2d4b26a3",
    "metadata": {},
    "outputs": [
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "Token indices sequence length is longer than the specified maximum sequence length for this model (3779 > 1024). Running this sequence through the model will result in indexing errors\n"
+      "Token indices sequence length is longer than the specified maximum sequence length for this model (6239 > 1024). Running this sequence through the model will result in indexing errors\n"
      ]
     }
    ],
@@ -1607,49 +1623,24 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 54,
+   "execution_count": 13,
    "id": "ce1769ed-b6e6-4c99-ab6d-7a4e567ddaa3",
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "(34, 67597, 5766.785714285715, 3167.0)"
+       "(76, 86560, 7774.184523809524, 4387.5)"
       ]
      },
-     "execution_count": 54,
+     "execution_count": 13,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
     "np.min(lengths),np.max(lengths),np.mean(lengths), np.median(lengths)"
    ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 10,
-   "id": "5c277ded-9567-44e5-9425-d5743af04d0e",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "(73846, 10425, 11253)"
-      ]
-     },
-     "execution_count": 10,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "train_csv = pd.read_csv(\"/Users/quert/Documents/GitHub/edit_NetKu/dataset/same_secs_insert_labeled/merged_train.csv\")\n",
-    "test_csv = pd.read_csv(\"/Users/quert/Documents/GitHub/edit_NetKu/dataset/same_secs_insert_labeled/merged_test.csv\")\n",
-    "val_csv = pd.read_csv(\"/Users/quert/Documents/GitHub/edit_NetKu/dataset/same_secs_insert_labeled/merged_val.csv\")\n",
-    "\n",
-    "len(train_csv), len(test_csv), len(val_csv)"
-   ]
   }
  ],
  "metadata": {