count_data_size.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "H3_txt_train_seqs_len: 11971\n",
      "H3_txt_test_seqs_len: 1497\n",
      "H3K4me1_txt_train_seqs_len: 25341\n",
      "H3K4me1_txt_test_seqs_len: 3168\n",
      "H3K4me2_txt_train_seqs_len: 24545\n",
      "H3K4me2_txt_test_seqs_len: 3069\n",
      "H3K4me3_txt_train_seqs_len: 29439\n",
      "H3K4me3_txt_test_seqs_len: 3680\n",
      "H3K9ac_txt_train_seqs_len: 22224\n",
      "H3K9ac_txt_test_seqs_len: 2779\n",
      "H3K14ac_txt_train_seqs_len: 26438\n",
      "H3K14ac_txt_test_seqs_len: 3305\n",
      "H3K36me3_txt_train_seqs_len: 27904\n",
      "H3K36me3_txt_test_seqs_len: 3488\n",
      "H3K79me3_txt_train_seqs_len: 23069\n",
      "H3K79me3_txt_test_seqs_len: 2884\n",
      "H4_txt_train_seqs_len: 11679\n",
      "H4_txt_test_seqs_len: 1461\n",
      "H4ac_txt_train_seqs_len: 27275\n",
      "H4ac_txt_test_seqs_len: 3410\n"
     ]
    }
   ],
   "source": [
    "from pathlib import Path\n",
    "\n",
    "split='train'\n",
    "dest_path = '/data/EMP'\n",
    "for dataset_name in ['H3_txt','H3K4me1_txt','H3K4me2_txt','H3K4me3_txt','H3K9ac_txt','H3K14ac_txt','H3K36me3_txt','H3K79me3_txt','H4_txt','H4ac_txt']:\n",
    "    for split in ['train', 'test']:\n",
    "# use Path object\n",
    "        base_path = Path(dest_path) / dataset_name / split\n",
    "\n",
    "        all_seqs = []\n",
    "        all_labels = []\n",
    "        label_mapper = {}\n",
    "\n",
    "        for i, x in enumerate(base_path.iterdir()):\n",
    "            label_mapper[x.stem] = i\n",
    "\n",
    "        for label_type in label_mapper.keys():\n",
    "            for path in (base_path / label_type).iterdir():\n",
    "                with open(path, \"r\") as f:\n",
    "                    content = f.read()\n",
    "                all_seqs.append(content)\n",
    "                all_labels.append(label_mapper[label_type])\n",
    "                \n",
    "        #print all_seqs length with details\n",
    "        print(f\"{dataset_name}_{split}_seqs_len: {len(all_seqs)}\")\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0_txt_train_seqs_len: 6478\n",
      "0_txt_test_seqs_len: 810\n",
      "1_txt_train_seqs_len: 53952\n",
      "1_txt_test_seqs_len: 6745\n",
      "2_txt_train_seqs_len: 2620\n",
      "2_txt_test_seqs_len: 328\n",
      "3_txt_train_seqs_len: 1904\n",
      "3_txt_test_seqs_len: 239\n",
      "4_txt_train_seqs_len: 15064\n",
      "4_txt_test_seqs_len: 1883\n"
     ]
    }
   ],
   "source": [
    "from pathlib import Path\n",
    "\n",
    "split='train'\n",
    "dest_path = '/data/mouse'\n",
    "for dataset_name in ['0_txt','1_txt','2_txt','3_txt','4_txt']:\n",
    "    for split in ['train', 'test']:\n",
    "# use Path object\n",
    "        base_path = Path(dest_path) / dataset_name / split\n",
    "\n",
    "        all_seqs = []\n",
    "        all_labels = []\n",
    "        label_mapper = {}\n",
    "\n",
    "        for i, x in enumerate(base_path.iterdir()):\n",
    "            label_mapper[x.stem] = i\n",
    "\n",
    "        for label_type in label_mapper.keys():\n",
    "            for path in (base_path / label_type).iterdir():\n",
    "                with open(path, \"r\") as f:\n",
    "                    content = f.read()\n",
    "                all_seqs.append(content)\n",
    "                all_labels.append(label_mapper[label_type])\n",
    "                \n",
    "        #print all_seqs length with details\n",
    "        print(f\"{dataset_name}_{split}_seqs_len: {len(all_seqs)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "prom_300_all_txt_train_seqs_len: 47356\n",
      "prom_300_all_txt_test_seqs_len: 5920\n",
      "prom_300_notata_txt_train_seqs_len: 42452\n",
      "prom_300_notata_txt_test_seqs_len: 5307\n",
      "prom_300_tata_txt_train_seqs_len: 4904\n",
      "prom_300_tata_txt_test_seqs_len: 613\n",
      "prom_core_all_txt_train_seqs_len: 47356\n",
      "prom_core_all_txt_test_seqs_len: 5920\n",
      "prom_core_notata_txt_train_seqs_len: 42452\n",
      "prom_core_notata_txt_test_seqs_len: 5307\n",
      "prom_core_tata_txt_train_seqs_len: 4904\n",
      "prom_core_tata_txt_test_seqs_len: 613\n"
     ]
    }
   ],
   "source": [
    "from pathlib import Path\n",
    "\n",
    "split='train'\n",
    "dest_path = '/data/prom'\n",
    "for dataset_name in ['prom_300_all_txt','prom_300_notata_txt','prom_300_tata_txt','prom_core_all_txt','prom_core_notata_txt','prom_core_tata_txt']:\n",
    "    for split in ['train', 'test']:\n",
    "# use Path object\n",
    "        base_path = Path(dest_path) / dataset_name / split\n",
    "\n",
    "        all_seqs = []\n",
    "        all_labels = []\n",
    "        label_mapper = {}\n",
    "\n",
    "        for i, x in enumerate(base_path.iterdir()):\n",
    "            label_mapper[x.stem] = i\n",
    "\n",
    "        for label_type in label_mapper.keys():\n",
    "            for path in (base_path / label_type).iterdir():\n",
    "                with open(path, \"r\") as f:\n",
    "                    content = f.read()\n",
    "                all_seqs.append(content)\n",
    "                all_labels.append(label_mapper[label_type])\n",
    "                \n",
    "        #print all_seqs length with details\n",
    "        print(f\"{dataset_name}_{split}_seqs_len: {len(all_seqs)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "reconstructed_txt_train_seqs_len: 36496\n",
      "reconstructed_txt_test_seqs_len: 4562\n"
     ]
    }
   ],
   "source": [
    "from pathlib import Path\n",
    "\n",
    "split='train'\n",
    "dest_path = '/data/splice'\n",
    "for dataset_name in ['reconstructed_txt']:\n",
    "    for split in ['train', 'test']:\n",
    "# use Path object\n",
    "        base_path = Path(dest_path) / dataset_name / split\n",
    "\n",
    "        all_seqs = []\n",
    "        all_labels = []\n",
    "        label_mapper = {}\n",
    "\n",
    "        for i, x in enumerate(base_path.iterdir()):\n",
    "            label_mapper[x.stem] = i\n",
    "\n",
    "        for label_type in label_mapper.keys():\n",
    "            for path in (base_path / label_type).iterdir():\n",
    "                with open(path, \"r\") as f:\n",
    "                    content = f.read()\n",
    "                all_seqs.append(content)\n",
    "                all_labels.append(label_mapper[label_type])\n",
    "                \n",
    "        #print all_seqs length with details\n",
    "        print(f\"{dataset_name}_{split}_seqs_len: {len(all_seqs)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0_txt_train_seqs_len: 32378\n",
      "0_txt_test_seqs_len: 1000\n",
      "1_txt_train_seqs_len: 30672\n",
      "1_txt_test_seqs_len: 1000\n",
      "2_txt_train_seqs_len: 19000\n",
      "2_txt_test_seqs_len: 1000\n",
      "3_txt_train_seqs_len: 27294\n",
      "3_txt_test_seqs_len: 1000\n",
      "4_txt_train_seqs_len: 19000\n",
      "4_txt_test_seqs_len: 1000\n"
     ]
    }
   ],
   "source": [
    "from pathlib import Path\n",
    "\n",
    "split='train'\n",
    "dest_path = '/data/tf'\n",
    "for dataset_name in ['0_txt','1_txt','2_txt','3_txt','4_txt']:\n",
    "    for split in ['train', 'test']:\n",
    "# use Path object\n",
    "        base_path = Path(dest_path) / dataset_name / split\n",
    "\n",
    "        all_seqs = []\n",
    "        all_labels = []\n",
    "        label_mapper = {}\n",
    "\n",
    "        for i, x in enumerate(base_path.iterdir()):\n",
    "            label_mapper[x.stem] = i\n",
    "\n",
    "        for label_type in label_mapper.keys():\n",
    "            for path in (base_path / label_type).iterdir():\n",
    "                with open(path, \"r\") as f:\n",
    "                    content = f.read()\n",
    "                all_seqs.append(content)\n",
    "                all_labels.append(label_mapper[label_type])\n",
    "                \n",
    "        #print all_seqs length with details\n",
    "        print(f\"{dataset_name}_{split}_seqs_len: {len(all_seqs)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "covid_txt_train_seqs_len: 73335\n",
      "covid_txt_test_seqs_len: 9168\n"
     ]
    }
   ],
   "source": [
    "from pathlib import Path\n",
    "\n",
    "split='train'\n",
    "dest_path = '/data/virus'\n",
    "for dataset_name in ['covid_txt']:\n",
    "    for split in ['train', 'test']:\n",
    "# use Path object\n",
    "        base_path = Path(dest_path) / dataset_name / split\n",
    "\n",
    "        all_seqs = []\n",
    "        all_labels = []\n",
    "        label_mapper = {}\n",
    "\n",
    "        for i, x in enumerate(base_path.iterdir()):\n",
    "            label_mapper[x.stem] = i\n",
    "\n",
    "        for label_type in label_mapper.keys():\n",
    "            for path in (base_path / label_type).iterdir():\n",
    "                with open(path, \"r\") as f:\n",
    "                    content = f.read()\n",
    "                all_seqs.append(content)\n",
    "                all_labels.append(label_mapper[label_type])\n",
    "                \n",
    "        #print all_seqs length with details\n",
    "        print(f\"{dataset_name}_{split}_seqs_len: {len(all_seqs)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0_txt_train_seqs_len: 41501\n",
      "0_txt_test_seqs_len: 11839\n"
     ]
    }
   ],
   "source": [
    "from pathlib import Path\n",
    "\n",
    "split='train'\n",
    "dest_path = '/data/promoter_prediction'\n",
    "for dataset_name in ['0_txt']:\n",
    "    for split in ['train', 'test']:\n",
    "# use Path object\n",
    "        base_path = Path(dest_path) / dataset_name / split\n",
    "\n",
    "        all_seqs = []\n",
    "        all_labels = []\n",
    "        label_mapper = {}\n",
    "\n",
    "        for i, x in enumerate(base_path.iterdir()):\n",
    "            label_mapper[x.stem] = i\n",
    "\n",
    "        for label_type in label_mapper.keys():\n",
    "            for path in (base_path / label_type).iterdir():\n",
    "                with open(path, \"r\") as f:\n",
    "                    content = f.read()\n",
    "                all_seqs.append(content)\n",
    "                all_labels.append(label_mapper[label_type])\n",
    "                \n",
    "        #print all_seqs length with details\n",
    "        print(f\"{dataset_name}_{split}_seqs_len: {len(all_seqs)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(146436,)\n",
      "(16270,)\n"
     ]
    }
   ],
   "source": [
    "#read np file\n",
    "import numpy as np\n",
    "\n",
    "np_file = '/data/splicing_prediction/train_seq.npy'\n",
    "\n",
    "data = np.load(np_file)\n",
    "\n",
    "print(data.shape)\n",
    "\n",
    "np_file_test='/data/splicing_prediction/test_seq.npy'\n",
    "\n",
    "data_test = np.load(np_file_test)\n",
    "\n",
    "print(data_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of lines in train data: 402297\n",
      "Number of lines in test data: 41187\n"
     ]
    }
   ],
   "source": [
    "#read txt file and count the number of lines\n",
    "\n",
    "train=open('/data/drosophila_enhancer_activity/Sequences_activity_train.txt','r')\n",
    "train_data=train.readlines()\n",
    "train.close()\n",
    "#print line number of train data\n",
    "print(\"Number of lines in train data:\",len(train_data))\n",
    "\n",
    "test=open('/data/drosophila_enhancer_activity/Sequences_activity_test.txt','r')\n",
    "test_data=test.readlines()\n",
    "test.close()\n",
    "#print line number of test data\n",
    "print(\"Number of lines in test data:\",len(test_data))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}