ali_twitter_final_text_features.json

{"paragraphs":[{"text":"%pyspark\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\n\nfrom pyspark.sql import SQLContext\nfrom pyspark.sql import functions as F\nfrom pyspark.sql import types as T\nfrom pyspark.sql import Row\nfrom pyspark.sql.window import Window\n\nfrom transformers import BertTokenizer\ntokenizer = BertTokenizer.from_pretrained('bert-base-multilingual-cased', do_lower_case=False)\n\n@F.udf(\"String\")\ndef decode_tokens(tokens):\n  return tokenizer.decode(tokens)\n\nsqc = SQLContext(sc)","user":"anonymous","dateUpdated":"2020-06-14T19:19:08+0000","config":{"colWidth":12,"fontSize":9,"enabled":true,"results":{},"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"editorMode":"ace/mode/python"},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[]},"apps":[],"jobName":"paragraph_1592036825963_14416857","id":"20200613-082705_91253728","dateCreated":"2020-06-13T08:27:05+0000","dateStarted":"2020-06-14T19:19:08+0000","dateFinished":"2020-06-14T19:19:09+0000","status":"FINISHED","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:17139"},{"text":"%pyspark\nroot_file_path = \"/nas_ssd_social_media_analytics/ali_twitter/final_dataset_12062020/temp/\"\ntraining_parquet_path = root_file_path+\"training_df\"\nvalidation_parquet_path = root_file_path+\"val_df\"\ntest_parquet_path = root_file_path+\"test_df\"\n\ntraining_df = sqc.read.parquet(training_parquet_path)\nvalidation_df = sqc.read.parquet(validation_parquet_path)\ntest_df = sqc.read.parquet(test_parquet_path)","user":"anonymous","dateUpdated":"2020-06-14T19:19:10+0000","config":{"colWidth":12,"fontSize":9,"enabled":true,"results":{},"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"editorMode":"ace/mode/python"},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[]},"apps":[],"jobName":"paragraph_1592036835731_-1527514350","id":"20200613-082715_2035782619","dateCreated":"2020-06-13T08:27:15+0000","dateStarted":"2020-06-14T19:19:10+0000","dateFinished":"2020-06-14T19:19:10+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:17140"},{"text":"%pyspark\ntraining_tweets = training_df.dropDuplicates([\"tweet_id\"])\\\n.select(\"tweet_id\",\n        decode_tokens(F.col(\"text_tokens\")).alias(\"text\"),\n        \"tweet_type\",\n        \"language\",\n        F.hour(F.to_timestamp(\"timestamp\")).alias(\"hour_tweet\"),\n        F.size(\"text_tokens\").alias(\"num_tokens\"),\n        F.when(F.col(\"hashtags\").isNull(), 0).otherwise(1).alias(\"has_hashtags\"),\n        F.when(F.col(\"present_media\").isNull(), 0).otherwise(1).alias(\"has_media\"), \n        F.when(F.col(\"present_links\").isNull(), 0).otherwise(1).alias(\"has_links\"))\n\n\n# training_tweets.write.parquet(root_file_path+\"training_tweets\")","user":"anonymous","dateUpdated":"2020-06-14T19:19:13+0000","config":{"colWidth":12,"fontSize":9,"enabled":true,"results":{},"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"editorMode":"ace/mode/python"},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[]},"apps":[],"jobName":"paragraph_1592036851650_-183212457","id":"20200613-082731_2098028395","dateCreated":"2020-06-13T08:27:31+0000","dateStarted":"2020-06-14T19:19:13+0000","dateFinished":"2020-06-14T19:19:14+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:17141"},{"text":"%pyspark\ntweet_is_in_top_daily_hashtag = training_df\\\n.dropDuplicates([\"tweet_id\"])\\\n.select(\"tweet_id\", \"timestamp\", F.explode(\"hashtags\").alias(\"hashtags_exploded\"))\\\n.withColumn(\"hashtag_duration\", F.max(\"timestamp\").over(Window.partitionBy(\"hashtags_exploded\")))\\\n.withColumn(\"hashtag_duration\", F.col(\"hashtag_duration\") - F.min(\"timestamp\").over(Window.partitionBy(\"hashtags_exploded\")))\\\n.withColumn(\"hashtag_duration2\", F.col(\"hashtag_duration\") / (24*3600))\\\n.withColumn(\"hashtag_duration3\", F.round(\"hashtag_duration2\"))\\\n.withColumn(\"is_in_top_daily_hashtags\", F.when( ((F.col(\"hashtag_duration3\") > 0) & (F.abs(F.col(\"hashtag_duration3\") - F.col(\"hashtag_duration2\")) < 0.007)), 1).otherwise(0))\\\n.select(\"tweet_id\", \"is_in_top_daily_hashtags\")\\\n.groupBy(\"tweet_id\").agg(F.max(\"is_in_top_daily_hashtags\").alias(\"is_in_top_daily_hashtags\"))\n\ntraining_tweets = training_tweets.join(tweet_is_in_top_daily_hashtag, \"tweet_id\", \"left_outer\")\\\n.withColumn(\"is_in_top_daily_hashtags\", F.when(F.col(\"is_in_top_daily_hashtags\").isNull(), 0).otherwise(F.col(\"is_in_top_daily_hashtags\")))","user":"anonymous","dateUpdated":"2020-06-14T19:19:17+0000","config":{"colWidth":12,"fontSize":9,"enabled":true,"results":{},"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"editorMode":"ace/mode/python"},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[]},"apps":[],"jobName":"paragraph_1592158840493_-947302548","id":"20200614-182040_916351753","dateCreated":"2020-06-14T18:20:40+0000","dateStarted":"2020-06-14T19:19:17+0000","dateFinished":"2020-06-14T19:19:18+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:17142"},{"text":"%pyspark\n\ntweet_is_in_top_daily_link = training_df\\\n.dropDuplicates([\"tweet_id\"])\\\n.select(\"tweet_id\", \"timestamp\", F.explode(\"present_links\").alias(\"present_links_exploded\"))\\\n.withColumn(\"link_duration\", F.max(\"timestamp\").over(Window.partitionBy(\"present_links_exploded\")))\\\n.withColumn(\"link_duration\", F.col(\"link_duration\") - F.min(\"timestamp\").over(Window.partitionBy(\"present_links_exploded\")))\\\n.withColumn(\"link_duration2\", F.col(\"link_duration\") / (24*3600))\\\n.withColumn(\"link_duration3\", F.round(\"link_duration2\"))\\\n.withColumn(\"is_in_top_daily_links\", F.when( ((F.col(\"link_duration3\") > 0) & (F.abs(F.col(\"link_duration3\") - F.col(\"link_duration2\")) < 0.007)), 1).otherwise(0))\\\n.select(\"tweet_id\", \"is_in_top_daily_links\")\\\n.groupBy(\"tweet_id\").agg(F.max(\"is_in_top_daily_links\").alias(\"is_in_top_daily_links\"))\n\ntraining_tweets = training_tweets.join(tweet_is_in_top_daily_link, \"tweet_id\", \"left_outer\")\\\n.withColumn(\"is_in_top_daily_links\", F.when(F.col(\"is_in_top_daily_links\").isNull(), 0).otherwise(F.col(\"is_in_top_daily_links\")))","user":"anonymous","dateUpdated":"2020-06-14T19:19:24+0000","config":{"colWidth":12,"fontSize":9,"enabled":true,"results":{},"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"editorMode":"ace/mode/python"},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[]},"apps":[],"jobName":"paragraph_1592158843585_-198970541","id":"20200614-182043_1948806389","dateCreated":"2020-06-14T18:20:43+0000","dateStarted":"2020-06-14T19:19:24+0000","dateFinished":"2020-06-14T19:19:24+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:17143"},{"text":"%pyspark\ntraining_tweets.write.parquet(root_file_path+\"training_tweets\")","user":"anonymous","dateUpdated":"2020-06-14T19:20:06+0000","config":{"colWidth":12,"fontSize":9,"enabled":true,"results":{},"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"editorMode":"ace/mode/python"},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[]},"apps":[],"jobName":"paragraph_1592162253512_732033549","id":"20200614-191733_2112916659","dateCreated":"2020-06-14T19:17:33+0000","dateStarted":"2020-06-14T19:20:06+0000","dateFinished":"2020-06-14T19:35:56+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:17144"},{"text":"%pyspark\ntraining_tweets = sqc.read.parquet(root_file_path+\"training_tweets\")","user":"anonymous","dateUpdated":"2020-06-14T19:20:12+0000","config":{"colWidth":12,"fontSize":9,"enabled":true,"results":{},"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"editorMode":"ace/mode/python"},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[]},"apps":[],"jobName":"paragraph_1592037166024_-675986762","id":"20200613-083246_2036467091","dateCreated":"2020-06-13T08:32:46+0000","dateStarted":"2020-06-14T19:20:12+0000","dateFinished":"2020-06-14T19:35:57+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:17145"},{"title":"Create TF-IDF vectors","text":"%pyspark\nfrom pyspark.ml import Pipeline\nfrom pyspark.ml.feature import Tokenizer, RegexTokenizer, StopWordsRemover, HashingTF, IDF, CountVectorizer\n\n# tokenizer = Tokenizer(inputCol=\"text\", outputCol=\"tokens\")\n# tmp = tokenizer.transform(text_d\n\nreTokenizer = RegexTokenizer(inputCol=\"text\", outputCol=\"words\", pattern=\"(?U)\\\\bhttps?://\\\\S*|#?\\\\b\\\\w+\\\\b\", gaps=False)\n# tmp = reTokenizer.transform(tmp)\n\nlangs = [\"english\", \"french\", \"spanish\", \"german\", \"finnish\", \"turkish\", \"english\", \"russian\", \"norwegian\", \"dutch\", \"danish\", \"hungarian\", \"italian\", \"swedish\", \"portuguese\"]\nmyStopWords = []\nfor i in langs:\n    myStopWords += StopWordsRemover.loadDefaultStopWords(i)\notherWordsToRemove = [\"cls\", \"sep\", \"unk\",  \"@\", \"rt\"]\nmyStopWords += otherWordsToRemove\n\nstopwordsRemover = StopWordsRemover(inputCol=reTokenizer.getOutputCol(), outputCol=\"tokens2\", stopWords=myStopWords)\n# tmp = stopwordsRemover.transform(tmp)\n\n# cv = CountVectorizer(inputCol=stopwordsRemover.getOutputCol(), outputCol=\"tf\")\n# model = cv.fit(df)\n\nhashingTF = HashingTF(inputCol=stopwordsRemover.getOutputCol(), outputCol=\"hashedTF\", numFeatures=16)\n# tmp = hashingTF.transform(tmp)\n\n# tf.cache()\n\nIDF = IDF(inputCol=hashingTF.getOutputCol(), outputCol=\"text_features\")\n# idf = IDF().fit(tmp)\n# tfidf = idf.transform(tf)\n\n# idfIgnore = IDF(minDocFreq=2).fit(tf)\n# tfidfIgnore = idfIgnore.transform(tf)\n\nstages = [reTokenizer, stopwordsRemover, hashingTF, IDF]\n\ntf_idf_pipeline = Pipeline(stages=stages)\n# tfidf_model = tf_idf_pipeline.fit(training_tweets)\n# tfidf_model.transform(training_tweets).select(\"tweet_id\", \"text_features\").show()","user":"anonymous","dateUpdated":"2020-06-14T19:20:15+0000","config":{"colWidth":12,"fontSize":9,"enabled":true,"results":{},"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"editorMode":"ace/mode/python","title":true},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[]},"apps":[],"jobName":"paragraph_1592038039651_1056462226","id":"20200613-084719_704568526","dateCreated":"2020-06-13T08:47:19+0000","dateStarted":"2020-06-14T19:35:57+0000","dateFinished":"2020-06-14T19:35:57+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:17146"},{"text":"%pyspark\nfrom pyspark.ml.feature import StringIndexer, OneHotEncoderEstimator, VectorAssembler, FeatureHasher\nfrom pyspark.ml import Pipeline\n\ncategoricalColumns = [\"tweet_type\", \"language\", \"hour_tweet\", \"has_hashtags\", \"has_media\", \"has_links\", \"is_in_top_daily_hashtags\", \"is_in_top_daily_links\"]\nstages = [] # stages in our Pipeline\nfor categoricalCol in categoricalColumns:\n    # Category Indexing with StringIndexer\n    stringIndexer = StringIndexer(inputCol=categoricalCol, outputCol=categoricalCol + \"Index\")\n    stages += [stringIndexer]\n    \n# assemblerInputs = [c + \"classVec\" for c in categoricalColumns]\n# assembler = VectorAssembler(inputCols=assemblerInputs, outputCol=\"featuresAssembled\")\n# stages += [assembler]\n\nfeatInputs = [c + \"Index\" for c in categoricalColumns]\nfeatHasher = FeatureHasher(numFeatures=16, inputCols=featInputs, outputCol=\"otherFeaturesHashed\", categoricalCols=featInputs)\nstages += [featHasher]\n\nother_tweet_features_pipline = Pipeline(stages=stages)\n# other_tweet_features = other_tweet_features_pipline.fit(training_tweets)","user":"anonymous","dateUpdated":"2020-06-14T19:20:32+0000","config":{"colWidth":12,"fontSize":9,"enabled":true,"results":{},"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"editorMode":"ace/mode/python"},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[]},"apps":[],"jobName":"paragraph_1592038860931_-444580734","id":"20200613-090100_2046421566","dateCreated":"2020-06-13T09:01:00+0000","dateStarted":"2020-06-14T19:35:57+0000","dateFinished":"2020-06-14T19:35:58+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:17147"},{"title":"Tweet features","text":"%pyspark\nassemble_tweet_features = VectorAssembler(inputCols=[\"text_features\", \"otherFeaturesHashed\"], outputCol=\"tweet_features\")\ncreate_tweet_features = Pipeline(stages=[tf_idf_pipeline, other_tweet_features_pipline, assemble_tweet_features]).fit(training_tweets)","user":"anonymous","dateUpdated":"2020-06-14T19:20:34+0000","config":{"colWidth":12,"fontSize":9,"enabled":true,"results":{},"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"editorMode":"ace/mode/python","title":true},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[]},"apps":[],"jobName":"paragraph_1592039902111_154101360","id":"20200613-091822_378393048","dateCreated":"2020-06-13T09:18:22+0000","dateStarted":"2020-06-14T19:35:58+0000","dateFinished":"2020-06-14T19:42:16+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:17148"},{"text":"%pyspark\ncreate_tweet_features.transform(training_tweets).select(\"tweet_id\", \"tweet_features\").show()","user":"anonymous","dateUpdated":"2020-06-14T19:20:53+0000","config":{"colWidth":12,"fontSize":9,"enabled":true,"results":{},"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"editorMode":"ace/mode/python"},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"+--------------------+--------------------+\n|            tweet_id|      tweet_features|\n+--------------------+--------------------+\n|000011A13832F3E13...|(32,[2,3,8,10,14,...|\n|000097495FAB6C041...|(32,[0,2,3,4,5,6,...|\n|0000FDBD4D9C8A9C4...|(32,[1,2,4,5,6,7,...|\n|000114CB0D3F0AAEE...|[2.51325001060600...|\n|00014A4A22D280DAC...|(32,[0,2,3,4,6,7,...|\n|000177B05B89EC5A1...|(32,[0,1,5,6,7,8,...|\n|00019DEBBD719CD8E...|(32,[0,2,4,5,6,8,...|\n|0001A11C7BB0545C9...|(32,[0,5,6,7,9,12...|\n|0001CA3AD87867F76...|(32,[0,1,2,3,4,5,...|\n|0001D878D7A5E0FD0...|(32,[0,4,7,8,9,13...|\n|00023E143F5F384A8...|(32,[0,1,3,4,5,6,...|\n|000241C0663453865...|(32,[6,10,11,12,1...|\n|000258E321631506C...|(32,[0,11,20,21,2...|\n|000280A572131D275...|(32,[2,9,10,18,20...|\n|0002958E4007541B1...|(32,[0,4,5,6,7,8,...|\n|0002E1AA9A7CF2429...|(32,[1,2,4,5,7,10...|\n|000334CC555BCE2F6...|(32,[0,2,3,5,6,7,...|\n|00037B3919134222F...|(32,[2,5,6,10,11,...|\n|00038A65AE8A68126...|(32,[0,5,6,7,11,1...|\n|0003B89FCD4994C81...|(32,[0,4,6,7,12,1...|\n+--------------------+--------------------+\nonly showing top 20 rows\n\n"}]},"apps":[],"jobName":"paragraph_1592042134032_1177193073","id":"20200613-095534_272019656","dateCreated":"2020-06-13T09:55:34+0000","dateStarted":"2020-06-14T19:35:58+0000","dateFinished":"2020-06-14T19:42:17+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:17149"},{"title":"Save create_tweet_features model","text":"%pyspark\ncreate_tweet_features.save(root_file_path+\"create_tweet_features_model\")","user":"anonymous","dateUpdated":"2020-06-14T19:44:23+0000","config":{"colWidth":12,"fontSize":9,"enabled":true,"results":{},"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"editorMode":"ace/mode/python","title":true},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[]},"apps":[],"jobName":"paragraph_1592041215100_640959301","id":"20200613-094015_699337491","dateCreated":"2020-06-13T09:40:15+0000","dateStarted":"2020-06-14T19:44:23+0000","dateFinished":"2020-06-14T19:44:26+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:17150"},{"title":"Load saved model","text":"%pyspark\nfrom pyspark.ml import PipelineModel\ncreate_tweet_features = PipelineModel.load(root_file_path+\"create_tweet_features_model\")\n# create_tweet_features.transform(training_tweets).select(\"tweet_id\", \"tweet_features\").show()","user":"anonymous","dateUpdated":"2020-06-14T19:45:15+0000","config":{"colWidth":12,"fontSize":9,"enabled":true,"results":{},"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"editorMode":"ace/mode/python","title":true},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[]},"apps":[],"jobName":"paragraph_1592041449689_-1832774642","id":"20200613-094409_1133237530","dateCreated":"2020-06-13T09:44:09+0000","dateStarted":"2020-06-14T19:45:15+0000","dateFinished":"2020-06-14T19:45:18+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:17151"},{"title":"Create tweet features for validation dataset","text":"%pyspark\nvalidation_tweets = validation_df.dropDuplicates([\"tweet_id\"])\\\n.select(\"tweet_id\",\n        decode_tokens(F.col(\"text_tokens\")).alias(\"text\"),\n        \"tweet_type\",\n        \"language\",\n        F.hour(F.to_timestamp(\"timestamp\")).alias(\"hour_tweet\"),\n        F.size(\"text_tokens\").alias(\"num_tokens\"),\n        F.when(F.col(\"hashtags\").isNull(), 0).otherwise(1).alias(\"has_hashtags\"),\n        F.when(F.col(\"present_media\").isNull(), 0).otherwise(1).alias(\"has_media\"), \n        F.when(F.col(\"present_links\").isNull(), 0).otherwise(1).alias(\"has_links\"))\n        \ntweet_is_in_top_daily_hashtag = validation_df\\\n.dropDuplicates([\"tweet_id\"])\\\n.select(\"tweet_id\", \"timestamp\", F.explode(\"hashtags\").alias(\"hashtags_exploded\"))\\\n.withColumn(\"hashtag_duration\", F.max(\"timestamp\").over(Window.partitionBy(\"hashtags_exploded\")))\\\n.withColumn(\"hashtag_duration\", F.col(\"hashtag_duration\") - F.min(\"timestamp\").over(Window.partitionBy(\"hashtags_exploded\")))\\\n.withColumn(\"hashtag_duration2\", F.col(\"hashtag_duration\") / (24*3600))\\\n.withColumn(\"hashtag_duration3\", F.round(\"hashtag_duration2\"))\\\n.withColumn(\"is_in_top_daily_hashtags\", F.when( ((F.col(\"hashtag_duration3\") > 0) & (F.abs(F.col(\"hashtag_duration3\") - F.col(\"hashtag_duration2\")) < 0.007)), 1).otherwise(0))\\\n.select(\"tweet_id\", \"is_in_top_daily_hashtags\")\\\n.groupBy(\"tweet_id\").agg(F.max(\"is_in_top_daily_hashtags\").alias(\"is_in_top_daily_hashtags\"))\n\nvalidation_tweets = validation_tweets.join(tweet_is_in_top_daily_hashtag, \"tweet_id\", \"left_outer\")\\\n.withColumn(\"is_in_top_daily_hashtags\", F.when(F.col(\"is_in_top_daily_hashtags\").isNull(), 0).otherwise(F.col(\"is_in_top_daily_hashtags\")))\n\ntweet_is_in_top_daily_link = validation_df\\\n.dropDuplicates([\"tweet_id\"])\\\n.select(\"tweet_id\", \"timestamp\", F.explode(\"present_links\").alias(\"present_links_exploded\"))\\\n.withColumn(\"link_duration\", F.max(\"timestamp\").over(Window.partitionBy(\"present_links_exploded\")))\\\n.withColumn(\"link_duration\", F.col(\"link_duration\") - F.min(\"timestamp\").over(Window.partitionBy(\"present_links_exploded\")))\\\n.withColumn(\"link_duration2\", F.col(\"link_duration\") / (24*3600))\\\n.withColumn(\"link_duration3\", F.round(\"link_duration2\"))\\\n.withColumn(\"is_in_top_daily_links\", F.when( ((F.col(\"link_duration3\") > 0) & (F.abs(F.col(\"link_duration3\") - F.col(\"link_duration2\")) < 0.007)), 1).otherwise(0))\\\n.select(\"tweet_id\", \"is_in_top_daily_links\")\\\n.groupBy(\"tweet_id\").agg(F.max(\"is_in_top_daily_links\").alias(\"is_in_top_daily_links\"))\n\nvalidation_tweets = validation_tweets.join(tweet_is_in_top_daily_link, \"tweet_id\", \"left_outer\")\\\n.withColumn(\"is_in_top_daily_links\", F.when(F.col(\"is_in_top_daily_links\").isNull(), 0).otherwise(F.col(\"is_in_top_daily_links\")))\n\nvalidation_tweets.write.parquet(root_file_path+\"validation_tweets\")\n\n# create_tweet_features.transform(validation_tweets).select(\"tweet_id\", \"tweet_features\").show()","user":"anonymous","dateUpdated":"2020-06-14T23:54:59+0000","config":{"colWidth":12,"fontSize":9,"enabled":true,"results":{},"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"editorMode":"ace/mode/python","title":true},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[]},"apps":[],"jobName":"paragraph_1592039618783_-164969878","id":"20200613-091338_795281872","dateCreated":"2020-06-13T09:13:38+0000","dateStarted":"2020-06-15T00:01:10+0000","dateFinished":"2020-06-15T00:05:48+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:17152"},{"title":"Create tweet features for validation dataset","text":"%pyspark\n\ntest_tweets = test_df.dropDuplicates([\"tweet_id\"])\\\n.select(\"tweet_id\",\n        decode_tokens(F.col(\"text_tokens\")).alias(\"text\"),\n        \"tweet_type\",\n        \"language\",\n        F.hour(F.to_timestamp(\"timestamp\")).alias(\"hour_tweet\"),\n        F.size(\"text_tokens\").alias(\"num_tokens\"),\n        F.when(F.col(\"hashtags\").isNull(), 0).otherwise(1).alias(\"has_hashtags\"),\n        F.when(F.col(\"present_media\").isNull(), 0).otherwise(1).alias(\"has_media\"), \n        F.when(F.col(\"present_links\").isNull(), 0).otherwise(1).alias(\"has_links\"))\n        \ntest_tweet_is_in_top_daily_hashtag = test_df\\\n.dropDuplicates([\"tweet_id\"])\\\n.select(\"tweet_id\", \"timestamp\", F.explode(\"hashtags\").alias(\"hashtags_exploded\"))\\\n.withColumn(\"hashtag_duration\", F.max(\"timestamp\").over(Window.partitionBy(\"hashtags_exploded\")))\\\n.withColumn(\"hashtag_duration\", F.col(\"hashtag_duration\") - F.min(\"timestamp\").over(Window.partitionBy(\"hashtags_exploded\")))\\\n.withColumn(\"hashtag_duration2\", F.col(\"hashtag_duration\") / (24*3600))\\\n.withColumn(\"hashtag_duration3\", F.round(\"hashtag_duration2\"))\\\n.withColumn(\"is_in_top_daily_hashtags\", F.when( ((F.col(\"hashtag_duration3\") > 0) & (F.abs(F.col(\"hashtag_duration3\") - F.col(\"hashtag_duration2\")) < 0.007)), 1).otherwise(0))\\\n.select(\"tweet_id\", \"is_in_top_daily_hashtags\")\\\n.groupBy(\"tweet_id\").agg(F.max(\"is_in_top_daily_hashtags\").alias(\"is_in_top_daily_hashtags\"))\n\ntest_tweets = test_tweets.join(test_tweet_is_in_top_daily_hashtag, \"tweet_id\", \"left_outer\")\\\n.withColumn(\"is_in_top_daily_hashtags\", F.when(F.col(\"is_in_top_daily_hashtags\").isNull(), 0).otherwise(F.col(\"is_in_top_daily_hashtags\")))\n\ntest_tweet_is_in_top_daily_link = test_df\\\n.dropDuplicates([\"tweet_id\"])\\\n.select(\"tweet_id\", \"timestamp\", F.explode(\"present_links\").alias(\"present_links_exploded\"))\\\n.withColumn(\"link_duration\", F.max(\"timestamp\").over(Window.partitionBy(\"present_links_exploded\")))\\\n.withColumn(\"link_duration\", F.col(\"link_duration\") - F.min(\"timestamp\").over(Window.partitionBy(\"present_links_exploded\")))\\\n.withColumn(\"link_duration2\", F.col(\"link_duration\") / (24*3600))\\\n.withColumn(\"link_duration3\", F.round(\"link_duration2\"))\\\n.withColumn(\"is_in_top_daily_links\", F.when( ((F.col(\"link_duration3\") > 0) & (F.abs(F.col(\"link_duration3\") - F.col(\"link_duration2\")) < 0.007)), 1).otherwise(0))\\\n.select(\"tweet_id\", \"is_in_top_daily_links\")\\\n.groupBy(\"tweet_id\").agg(F.max(\"is_in_top_daily_links\").alias(\"is_in_top_daily_links\"))\n\ntest_tweets = test_tweets.join(test_tweet_is_in_top_daily_link, \"tweet_id\", \"left_outer\")\\\n.withColumn(\"is_in_top_daily_links\", F.when(F.col(\"is_in_top_daily_links\").isNull(), 0).otherwise(F.col(\"is_in_top_daily_links\")))\n\ntest_tweets.write.parquet(root_file_path+\"test_tweets\")\n\n# create_tweet_features.transform(test_tweets).select(\"tweet_id\", \"tweet_features\").show()","user":"anonymous","dateUpdated":"2020-06-14T23:54:56+0000","config":{"colWidth":12,"fontSize":9,"enabled":true,"results":{},"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"editorMode":"ace/mode/python","title":true},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[]},"apps":[],"jobName":"paragraph_1592042408839_472215561","id":"20200613-100008_1676943236","dateCreated":"2020-06-13T10:00:08+0000","dateStarted":"2020-06-14T23:54:56+0000","dateFinished":"2020-06-15T00:03:28+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:17153"},{"text":"%pyspark\n","user":"anonymous","dateUpdated":"2020-06-13T09:47:30+0000","config":{"colWidth":12,"fontSize":9,"enabled":true,"results":{},"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"editorMode":"ace/mode/python"},"settings":{"params":{},"forms":{}},"apps":[],"jobName":"paragraph_1592041650037_1147931756","id":"20200613-094730_471884988","dateCreated":"2020-06-13T09:47:30+0000","status":"READY","progressUpdateIntervalMs":500,"$$hashKey":"object:17154"}],"name":"ali/twitter/final/text_features","id":"2FBZRESSS","noteParams":{},"noteForms":{},"angularObjects":{"md:shared_process":[],"sh:shared_process":[],"spark:shared_process":[]},"config":{"isZeppelinNotebookCronEnable":false,"looknfeel":"default","personalizedMode":"false"},"info":{}}