Skip to content
This repository was archived by the owner on Feb 28, 2024. It is now read-only.

Separate title name of the text from the title received from Kavitakosh #77

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open

Separate title name of the text from the title received from Kavitakosh #77

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
3891ffe
Separate title name of the text from the title received from Kavitakosh
Apr 15, 2019
7041e8d
Add script that can clean crawled data from Kavitakosh.org
Jun 30, 2019
72cbba9
Cleaned crawled data
Jun 30, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
24 changes: 24 additions & 0 deletions elasticsearch/data/clean_crawled_data.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
#!/usr/bin/env python
# The purpose of this script is to fix bug https://github.com/Code4Nepal/akshara-project/issues/76
# Kavitakosh.org changed their format and started including author name in their title.
# The crawling script has been fixed too. This script helps to clean existing data so we do not need to re-index just for this reason.

import os, json

path_to_json = 'crawled/kavitakosh/'
json_files = []

# Find author folders in the folder and add all the json files inside it
for folder in os.listdir(path_to_json):
files = [os.path.join(path_to_json, folder, pos_json) for pos_json in os.listdir(path_to_json + folder) if pos_json.endswith('.json') and not 'undetected.json' in pos_json]
json_files.extend(files)

for json_file in json_files:
with open(json_file, 'r+') as outfile:
data = json.load(outfile)
tmp = data["title"].split('/')[0]
data["title"] = tmp

outfile.seek(0) # rewind
json.dump(data, outfile)
outfile.truncate()
2 changes: 1 addition & 1 deletion elasticsearch/data/crawled/kavitakosh/अखण्डदिलदास/श्री राग आरती .json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"author": "\u0905\u0916\u0923\u094d\u0921\u0926\u093f\u0932\u0926\u093e\u0938", "genre": "\u0915\u0935\u093f\u0924\u093e", "source_link": "http://kavitakosh.org/kk/%E0%A4%85%E0%A4%96%E0%A4%A3%E0%A5%8D%E0%A4%A1%E0%A4%A6%E0%A4%BF%E0%A4%B2%E0%A4%A6%E0%A4%BE%E0%A4%B8", "source": "\u0928\u0947\u092a\u093e\u0932\u0940 - \u0915\u0935\u093f\u0924\u093e \u0915\u094b\u0936", "title": "\u0936\u094d\u0930\u0940 \u0930\u093e\u0917 \u0906\u0930\u0924\u0940 / \u0905\u0916\u0923\u094d\u0921\u0926\u093f\u0932\u0926\u093e\u0938", "text": "\u0906\u0930\u0924\u0940 \u0932\u093e\u0909 \u0938\u093e\u0927\u0941 \u0906\u0930\u0924\u0940 \u0932\u093e\u0909 \u090f\u0915 \u0924\u0924\u094d\u0924\u0941 \u092a\u093e\u0930 \u092c\u094d\u0930\u092e\u094d\u0939\u093e\u0915\u0947\n\u0906\u0930\u0924\u0940 \u0932\u093e\u0909 \u0964\u0964\n\u091b\u0947\u0930 \u0905\u091b\u0947\u0930\u0938\u0930\u0941\u092a\u0928 \u092a\u093e\u0909 \u0926\u0942\u0943\u0927 \u0938\u092e\u093e\u0927\u093f \u092a\u0930\u092e\u094d\u092a\u0926 \u092a\u093e\u0909\u0901 \u0964\u0964\u0906\u0930\u0924\u093f\u0964\u0964\u0967\u0964\u0964\n\u0950 \u0905\u0915\u093e\u0930 \u0905\u093e\u0930 \u0909\u0915\u093e\u0930 \u092e\u0915\u093e\u0930 \u092e\u093f\u0932\u093e\u0908 \u0964\u0964\n\u0924\u094d\u0930\u0940\u0915\u0941\u091f\u093f\u0915\u0947 \u0935\u093f\u091a\u092e\u0947 \u091c\u094b\u0924\u093f \u0938\u0930\u0941\u092a \u0932\u0937\u0909 \u0906\u0907 \u0930 \u0964\u0964\u0906\u0930\u0924\u093f\u0964\u0964\u0968\u0964\u0964\n\u0926\u0943\u0927 \u0938\u092e\u093e\u0927\u093f \u092a\u094d\u0930\u0926\u0915\u094d\u0937 \u092a\u093e\u0909\n\u092d\u091c\u0928 \u092d\u0915\u094d\u0924 \u0915\u0930\u093f \u0939\u0930\u093f \u0938\u0902\u092e\u093f\u0932\u0940 \u091c\u093e\u0909 \u0964\u0964\u0906\u0930\u0924\u093f\u0964\u0964\u0969\u0964\u0964\n\u0917\u0917\u0928 \u0926\u0943\u0936\u094d\u0932 \u0917\u0927\u0947\u0926\u0941 \u0928\u0940\u091c \u0917\u093e\u0909\n\u0927\u0928\u0902\u091c \u092f\u0915 \u0915\u092e\u0932 \u092a\u093e\u0930 \u092a\u093e\u0909 \u0964\u0964\u0906\u0930\u0924\u093f\u0964\u0964\u096a\u0964\u0964\n\u0905\u0928\u093e\u0926\u093f \u0915\u0943\u0937\u094d\u0923 \u0938\u093f\u0930 \u091a\u0930\u093e\u0909\n\u0939\u0902\u0938\u0938 \u0938\u0930\u0941\u092a \u092a\u093e\u0930\u0924\u0930\u093f \u091c\u093e\u0909 \u0964\u0964\u0906\u0930\u0924\u093f\u0964\u0964\u096b\u0964\u0964\n\u0915\u0928\u094d\u091a\u0928 \u0938\u0930\u0941\u092a \u0938\u0924\u0947\u092f\u0930\u094d \u0938\u092e\u093e\u0909\n\u0927\u093f\u0930\u0947 \u0938\u0941\u092d\u0915\u094b \u092e\u0941\u0915\u094d\u0924\u093f \u092a\u093e\u0909 \u0964\u0964\u0906\u0930\u0924\u093f\u0964\u0964\u096c\u0964\u0964\n\u0926\u093e\u0938 \u0905\u0937\u0921\u0926\u093f\u0932 \u0906\u0930\u0924\u093f \u0932\u093e\u0909\n\u0939\u0930\u093f \u0924\u0928 \u092d\u0915\u094d\u0924\u093f \u0938\u0930 \u0928\u0930 \u0927\u093e\u092e \u092a\u0930\u0902 \u0927\u093e\u092e \u092a\u093e\u0909\u0902 \u0964\u0964\u0906\u0930\u0924\u093f\u0964\u0964\u096d\u0964\u0964\n\n\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0'", "lang": "\u0928\u0947\u092a\u093e\u0932\u0940"}
{"author": "\u0905\u0916\u0923\u094d\u0921\u0926\u093f\u0932\u0926\u093e\u0938", "genre": "\u0915\u0935\u093f\u0924\u093e", "source_link": "http://kavitakosh.org/kk/%E0%A4%85%E0%A4%96%E0%A4%A3%E0%A5%8D%E0%A4%A1%E0%A4%A6%E0%A4%BF%E0%A4%B2%E0%A4%A6%E0%A4%BE%E0%A4%B8", "source": "\u0928\u0947\u092a\u093e\u0932\u0940 - \u0915\u0935\u093f\u0924\u093e \u0915\u094b\u0936", "title": "\u0936\u094d\u0930\u0940 \u0930\u093e\u0917 \u0906\u0930\u0924\u0940 ", "text": "\u0906\u0930\u0924\u0940 \u0932\u093e\u0909 \u0938\u093e\u0927\u0941 \u0906\u0930\u0924\u0940 \u0932\u093e\u0909 \u090f\u0915 \u0924\u0924\u094d\u0924\u0941 \u092a\u093e\u0930 \u092c\u094d\u0930\u092e\u094d\u0939\u093e\u0915\u0947\n\u0906\u0930\u0924\u0940 \u0932\u093e\u0909 \u0964\u0964\n\u091b\u0947\u0930 \u0905\u091b\u0947\u0930\u0938\u0930\u0941\u092a\u0928 \u092a\u093e\u0909 \u0926\u0942\u0943\u0927 \u0938\u092e\u093e\u0927\u093f \u092a\u0930\u092e\u094d\u092a\u0926 \u092a\u093e\u0909\u0901 \u0964\u0964\u0906\u0930\u0924\u093f\u0964\u0964\u0967\u0964\u0964\n\u0950 \u0905\u0915\u093e\u0930 \u0905\u093e\u0930 \u0909\u0915\u093e\u0930 \u092e\u0915\u093e\u0930 \u092e\u093f\u0932\u093e\u0908 \u0964\u0964\n\u0924\u094d\u0930\u0940\u0915\u0941\u091f\u093f\u0915\u0947 \u0935\u093f\u091a\u092e\u0947 \u091c\u094b\u0924\u093f \u0938\u0930\u0941\u092a \u0932\u0937\u0909 \u0906\u0907 \u0930 \u0964\u0964\u0906\u0930\u0924\u093f\u0964\u0964\u0968\u0964\u0964\n\u0926\u0943\u0927 \u0938\u092e\u093e\u0927\u093f \u092a\u094d\u0930\u0926\u0915\u094d\u0937 \u092a\u093e\u0909\n\u092d\u091c\u0928 \u092d\u0915\u094d\u0924 \u0915\u0930\u093f \u0939\u0930\u093f \u0938\u0902\u092e\u093f\u0932\u0940 \u091c\u093e\u0909 \u0964\u0964\u0906\u0930\u0924\u093f\u0964\u0964\u0969\u0964\u0964\n\u0917\u0917\u0928 \u0926\u0943\u0936\u094d\u0932 \u0917\u0927\u0947\u0926\u0941 \u0928\u0940\u091c \u0917\u093e\u0909\n\u0927\u0928\u0902\u091c \u092f\u0915 \u0915\u092e\u0932 \u092a\u093e\u0930 \u092a\u093e\u0909 \u0964\u0964\u0906\u0930\u0924\u093f\u0964\u0964\u096a\u0964\u0964\n\u0905\u0928\u093e\u0926\u093f \u0915\u0943\u0937\u094d\u0923 \u0938\u093f\u0930 \u091a\u0930\u093e\u0909\n\u0939\u0902\u0938\u0938 \u0938\u0930\u0941\u092a \u092a\u093e\u0930\u0924\u0930\u093f \u091c\u093e\u0909 \u0964\u0964\u0906\u0930\u0924\u093f\u0964\u0964\u096b\u0964\u0964\n\u0915\u0928\u094d\u091a\u0928 \u0938\u0930\u0941\u092a \u0938\u0924\u0947\u092f\u0930\u094d \u0938\u092e\u093e\u0909\n\u0927\u093f\u0930\u0947 \u0938\u0941\u092d\u0915\u094b \u092e\u0941\u0915\u094d\u0924\u093f \u092a\u093e\u0909 \u0964\u0964\u0906\u0930\u0924\u093f\u0964\u0964\u096c\u0964\u0964\n\u0926\u093e\u0938 \u0905\u0937\u0921\u0926\u093f\u0932 \u0906\u0930\u0924\u093f \u0932\u093e\u0909\n\u0939\u0930\u093f \u0924\u0928 \u092d\u0915\u094d\u0924\u093f \u0938\u0930 \u0928\u0930 \u0927\u093e\u092e \u092a\u0930\u0902 \u0927\u093e\u092e \u092a\u093e\u0909\u0902 \u0964\u0964\u0906\u0930\u0924\u093f\u0964\u0964\u096d\u0964\u0964\n\n\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0'", "lang": "\u0928\u0947\u092a\u093e\u0932\u0940"}
2 changes: 1 addition & 1 deletion elasticsearch/data/crawled/kavitakosh/अगमदिलदास/राग निर्गुण .json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"author": "\u0905\u0917\u092e\u0926\u093f\u0932\u0926\u093e\u0938", "genre": "\u0915\u0935\u093f\u0924\u093e", "source_link": "http://kavitakosh.org/kk/%E0%A4%85%E0%A4%97%E0%A4%AE%E0%A4%A6%E0%A4%BF%E0%A4%B2%E0%A4%A6%E0%A4%BE%E0%A4%B8", "source": "\u0928\u0947\u092a\u093e\u0932\u0940 - \u0915\u0935\u093f\u0924\u093e \u0915\u094b\u0936", "title": "\u0930\u093e\u0917 \u0928\u093f\u0930\u094d\u0917\u0941\u0923 / \u0905\u0917\u092e\u0926\u093f\u0932\u0926\u093e\u0938", "text": "\u0930\u093e\u092e \u0939\u0930\u093f \u092d\u091c\u0940 \u0909\u0924\u093e\u0930\u0947 \u0928\u0926\u093f\u092f\u093e \u092a\u093e\u0930\u0915\u0947 \u0918\u0930 \u092e\u094b\u0939\u0928 \u092e\u093e\u092f\u093e \u0964\u0964\u091f\u0947\u0915\u0964\u0964\n\u092e\u094b\u0939\u0915\u094b \u091c\u0928\u094d\u091c\u0940\u0930 \u0924\u094b\u0921\u0940 \u0924\u0939\u093e\u0901 \u092e\u093e\u092f\u093e \u092d\u0941\u0932 \u0938\u094c \u092d\u0941\u0932\u0947 \u0906\u091c\n\u092e\u0924 \u092d\u0942\u0932\u094b \u0905\u092e\u0930 \u0939\u093e\u091f \u0928 \u0917\u0941\u092e\u093e\u090a, \u092e\u094b\u0939\u0928 \u092e\u093e\u092f\u093e \u0964\u0964\u0967\u0964\u0964\n\u092e\u094b\u0939\u0915\u0947 \u092c\u093e\u0926\u0932\u0941 \u092d\u0930\u0947\u090a \u092f\u0939\u093e\u0901 \u0924\u0939\u093e\u0901 \u0916\u0947\u0932 \u0938\u094b \u0916\u0947\u0932\n\u092c\u0928\u0940 \u0938\u092c \u0906\u0908\n\u091a\u0932\u093f\u090f \u0938\u0916\u0947\u0930 \u0905\u092e\u094d\u092e\u0930 \u0927\u093e\u092e\u092e\u093e \u091c\u093e\u090a\u0901, \u092e\u094b\u0939\u0928 \u092e\u093e\u092f\u093e \u0964\u0964\u0968\u0964\u0964\n\u0938\u093e\u0901\u091a\u093e\u0915\u094b \u0938\u093e\u0901\u091a\u0940 \u0938\u093e\u0947\u0939\u0940 \u092e\u0928 \u092c\u093e\u0901\u0927\u0940 \u091a\u0932\u093f\u092f \u0936\u093e\u0928\u094d\u0924 \u0935\u0939\u093f \u0926\u0947\u0936\n\u091c\u0939\u093e\u0901 \u0906\u0916\u0947\u091f \u091a\u093e\u0939\u0947 \u0928\u0940\u091c \u091a\u093e\u092e\u094d\u092e\u093e \u091c\u093e\u0909\u0901, \u092e\u094b\u0939\u0928 \u092e\u093e\u092f\u093e \u0964\u0964\u0969\u0964\u0964\n\u0938\u0941\u0930\u0924 \u0915\u0940 \u091a\u093e\u0939\u093e\u0930\u0940 \u0932\u094d\u092f\u094c \u0939\u093e\u0924 \u0928\u093e\u092e\u0936\u093f \u0938\u0941\u0928,\n\u0924\u0941\u0908 \u0928\u0924\u093e\u0928 \u0921\u0917\u092e\u0917 \u0928\u0939\u093f \u0938\u0941\u0930\u0924 \u0938\u093e\u0927\u0940\n\u092a\u0930\u092e\u094d \u0927\u093e\u092e\u094d\u092e\u093e \u091c\u093e\u0928\u0941 \u092f\u0939\u093f \u0930\u0940\u0924 \u092a\u093e\u0930\u0940, \u092e\u094b\u0939\u0928 \u092e\u093e\u092f\u093e \u0964\u0964\u096a\u0964\u0964\n\u0915\u0939\u0947 \u0926\u093e\u0938 \u0905\u0917\u092e\u0926\u093f\u0932 \u0938\u0941\u0928\u094b \u092d\u093e\u0908 \u0938\u093e\u0927\u0941\n\u091a\u093e\u0939\u0947 \u0935\u093f\u091a\u093e\u0939\u0947 \u0928\u093e\u0925\u0932\u0940 \u0913\u0918\u094b \u0909\u0920\u0940 \u0939\u0947\u0930\u0941\u0902 \u0926\u0947\u0936\n\u092a\u0930\u092e \u0927\u093e\u092e \u091c\u093e\u0928 \u092a\u0930\u0940 \u092a\u0941\u0922\u094b \u092e\u094b\u0939\u0928 \u092e\u093e\u092f\u093e", "lang": "\u0928\u0947\u092a\u093e\u0932\u0940"}
{"author": "\u0905\u0917\u092e\u0926\u093f\u0932\u0926\u093e\u0938", "genre": "\u0915\u0935\u093f\u0924\u093e", "source_link": "http://kavitakosh.org/kk/%E0%A4%85%E0%A4%97%E0%A4%AE%E0%A4%A6%E0%A4%BF%E0%A4%B2%E0%A4%A6%E0%A4%BE%E0%A4%B8", "source": "\u0928\u0947\u092a\u093e\u0932\u0940 - \u0915\u0935\u093f\u0924\u093e \u0915\u094b\u0936", "title": "\u0930\u093e\u0917 \u0928\u093f\u0930\u094d\u0917\u0941\u0923 ", "text": "\u0930\u093e\u092e \u0939\u0930\u093f \u092d\u091c\u0940 \u0909\u0924\u093e\u0930\u0947 \u0928\u0926\u093f\u092f\u093e \u092a\u093e\u0930\u0915\u0947 \u0918\u0930 \u092e\u094b\u0939\u0928 \u092e\u093e\u092f\u093e \u0964\u0964\u091f\u0947\u0915\u0964\u0964\n\u092e\u094b\u0939\u0915\u094b \u091c\u0928\u094d\u091c\u0940\u0930 \u0924\u094b\u0921\u0940 \u0924\u0939\u093e\u0901 \u092e\u093e\u092f\u093e \u092d\u0941\u0932 \u0938\u094c \u092d\u0941\u0932\u0947 \u0906\u091c\n\u092e\u0924 \u092d\u0942\u0932\u094b \u0905\u092e\u0930 \u0939\u093e\u091f \u0928 \u0917\u0941\u092e\u093e\u090a, \u092e\u094b\u0939\u0928 \u092e\u093e\u092f\u093e \u0964\u0964\u0967\u0964\u0964\n\u092e\u094b\u0939\u0915\u0947 \u092c\u093e\u0926\u0932\u0941 \u092d\u0930\u0947\u090a \u092f\u0939\u093e\u0901 \u0924\u0939\u093e\u0901 \u0916\u0947\u0932 \u0938\u094b \u0916\u0947\u0932\n\u092c\u0928\u0940 \u0938\u092c \u0906\u0908\n\u091a\u0932\u093f\u090f \u0938\u0916\u0947\u0930 \u0905\u092e\u094d\u092e\u0930 \u0927\u093e\u092e\u092e\u093e \u091c\u093e\u090a\u0901, \u092e\u094b\u0939\u0928 \u092e\u093e\u092f\u093e \u0964\u0964\u0968\u0964\u0964\n\u0938\u093e\u0901\u091a\u093e\u0915\u094b \u0938\u093e\u0901\u091a\u0940 \u0938\u093e\u0947\u0939\u0940 \u092e\u0928 \u092c\u093e\u0901\u0927\u0940 \u091a\u0932\u093f\u092f \u0936\u093e\u0928\u094d\u0924 \u0935\u0939\u093f \u0926\u0947\u0936\n\u091c\u0939\u093e\u0901 \u0906\u0916\u0947\u091f \u091a\u093e\u0939\u0947 \u0928\u0940\u091c \u091a\u093e\u092e\u094d\u092e\u093e \u091c\u093e\u0909\u0901, \u092e\u094b\u0939\u0928 \u092e\u093e\u092f\u093e \u0964\u0964\u0969\u0964\u0964\n\u0938\u0941\u0930\u0924 \u0915\u0940 \u091a\u093e\u0939\u093e\u0930\u0940 \u0932\u094d\u092f\u094c \u0939\u093e\u0924 \u0928\u093e\u092e\u0936\u093f \u0938\u0941\u0928,\n\u0924\u0941\u0908 \u0928\u0924\u093e\u0928 \u0921\u0917\u092e\u0917 \u0928\u0939\u093f \u0938\u0941\u0930\u0924 \u0938\u093e\u0927\u0940\n\u092a\u0930\u092e\u094d \u0927\u093e\u092e\u094d\u092e\u093e \u091c\u093e\u0928\u0941 \u092f\u0939\u093f \u0930\u0940\u0924 \u092a\u093e\u0930\u0940, \u092e\u094b\u0939\u0928 \u092e\u093e\u092f\u093e \u0964\u0964\u096a\u0964\u0964\n\u0915\u0939\u0947 \u0926\u093e\u0938 \u0905\u0917\u092e\u0926\u093f\u0932 \u0938\u0941\u0928\u094b \u092d\u093e\u0908 \u0938\u093e\u0927\u0941\n\u091a\u093e\u0939\u0947 \u0935\u093f\u091a\u093e\u0939\u0947 \u0928\u093e\u0925\u0932\u0940 \u0913\u0918\u094b \u0909\u0920\u0940 \u0939\u0947\u0930\u0941\u0902 \u0926\u0947\u0936\n\u092a\u0930\u092e \u0927\u093e\u092e \u091c\u093e\u0928 \u092a\u0930\u0940 \u092a\u0941\u0922\u094b \u092e\u094b\u0939\u0928 \u092e\u093e\u092f\u093e", "lang": "\u0928\u0947\u092a\u093e\u0932\u0940"}
Loading