first commit

Abiral-2724 · Jul 27, 2024 · 2895127 · 2895127
commit 2895127
Show file tree

Hide file tree

Showing 89 changed files with 480 additions and 0 deletions.
diff --git a/.github/workflows/main_pwengimagescrap.yml b/.github/workflows/main_pwengimagescrap.yml
@@ -0,0 +1,63 @@
+# Docs for the Azure Web Apps Deploy action: https://github.com/Azure/webapps-deploy
+# More GitHub Actions for Azure: https://github.com/Azure/actions
+# More info on Python, GitHub Actions, and Azure App Service: https://aka.ms/python-webapps-actions
+
+name: Build and deploy Python app to Azure Web App - pwengimagescrap
+
+on:
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Set up Python version
+        uses: actions/setup-python@v1
+        with:
+          python-version: '3.7'
+
+      - name: Create and start virtual environment
+        run: |
+          python -m venv venv
+          source venv/bin/activate
+      
+      - name: Install dependencies
+        run: pip install -r requirements.txt
+
+      # Optional: Add step to run tests here (PyTest, Django test suites, etc.)
+
+      - name: Upload artifact for deployment jobs
+        uses: actions/upload-artifact@v2
+        with:
+          name: python-app
+          path: |
+            . 
+            !venv/
+
+  deploy:
+    runs-on: ubuntu-latest
+    needs: build
+    environment:
+      name: 'Production'
+      url: ${{ steps.deploy-to-webapp.outputs.webapp-url }}
+
+    steps:
+      - name: Download artifact from build job
+        uses: actions/download-artifact@v2
+        with:
+          name: python-app
+          path: .
+
+      - name: 'Deploy to Azure Web App'
+        uses: azure/webapps-deploy@v2
+        id: deploy-to-webapp
+        with:
+          app-name: 'pwengimagescrap'
+          slot-name: 'Production'
+          publish-profile: ${{ secrets.AZUREAPPSERVICE_PUBLISHPROFILE_217D42C8DFF04DA7B4F63AD17D93911A }}
diff --git a/app.py b/app.py
@@ -0,0 +1,77 @@
+from flask import Flask, render_template, request,jsonify
+from flask_cors import CORS,cross_origin
+import requests
+from bs4 import BeautifulSoup
+from urllib.request import urlopen as uReq
+import logging
+import pymongo
+logging.basicConfig(filename="scrapper.log" , level=logging.INFO)
+import os
+
+app = Flask(__name__)
+
+@app.route("/", methods = ['GET'])
+def homepage():
+    return render_template("index.html")
+
+@app.route("/review" , methods = ['POST' , 'GET'])
+def index():
+    if request.method == 'POST':
+                try:
+
+                    # query to search for images
+                    query = request.form['content'].replace(" ","")
+
+                            # directory to store downloaded images
+                    save_directory = "images/"
+
+                            # create the directory if it doesn't exist
+                    if not os.path.exists(save_directory):
+                        os.makedirs(save_directory)
+
+
+
+                            # fake user agent to avoid getting blocked by Google
+                    headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36"}
+
+                            # fetch the search results page
+                    response = requests.get(f"https://www.google.com/search?q={query}&sxsrf=AJOqlzUuff1RXi2mm8I_OqOwT9VjfIDL7w:1676996143273&source=lnms&tbm=isch&sa=X&ved=2ahUKEwiq-qK7gaf9AhXUgVYBHYReAfYQ_AUoA3oECAEQBQ&biw=1920&bih=937&dpr=1#imgrc=1th7VhSesfMJ4M")
+
+
+                            # parse the HTML using BeautifulSoup
+                    soup = BeautifulSoup(response.content, "html.parser")
+
+                            # find all img tags
+                    image_tags = soup.find_all("img")
+
+                            # download each image and save it to the specified directory
+                    del image_tags[0]
+                    img_data=[]
+                    for index,image_tag in enumerate(image_tags):
+                                # get the image source URL
+                                image_url = image_tag['src']
+                                #print(image_url)
+
+                                # send a request to the image URL and save the image
+                                image_data = requests.get(image_url).content
+                                mydict={"Index":index,"Image":image_data}
+                                img_data.append(mydict)
+                                with open(os.path.join(save_directory, f"{query}_{image_tags.index(image_tag)}.jpg"), "wb") as f:
+                                    f.write(image_data)
+                    client = pymongo.MongoClient("mongodb+srv://snshrivas:[email protected]/?retryWrites=true&w=majority")
+                    db = client['image_scrap']
+                    review_col = db['image_scrap_data']
+                    review_col.insert_many(img_data)          
+
+                    return "image laoded"
+                except Exception as e:
+                    logging.info(e)
+                    return 'something is wrong'
+            # return render_template('results.html')
+
+    else:
+        return render_template('index.html')
+
+
+if __name__ == "__main__":
+    app.run(host='0.0.0.0', port=8000)
diff --git a/images/elonmusk_0.jpg b/images/elonmusk_0.jpg
diff --git a/images/elonmusk_1.jpg b/images/elonmusk_1.jpg
diff --git a/images/elonmusk_10.jpg b/images/elonmusk_10.jpg
diff --git a/images/elonmusk_11.jpg b/images/elonmusk_11.jpg
diff --git a/images/elonmusk_12.jpg b/images/elonmusk_12.jpg
diff --git a/images/elonmusk_13.jpg b/images/elonmusk_13.jpg
diff --git a/images/elonmusk_14.jpg b/images/elonmusk_14.jpg
diff --git a/images/elonmusk_15.jpg b/images/elonmusk_15.jpg
diff --git a/images/elonmusk_16.jpg b/images/elonmusk_16.jpg
diff --git a/images/elonmusk_17.jpg b/images/elonmusk_17.jpg
diff --git a/images/elonmusk_18.jpg b/images/elonmusk_18.jpg
diff --git a/images/elonmusk_19.jpg b/images/elonmusk_19.jpg
diff --git a/images/elonmusk_2.jpg b/images/elonmusk_2.jpg
diff --git a/images/elonmusk_3.jpg b/images/elonmusk_3.jpg
diff --git a/images/elonmusk_4.jpg b/images/elonmusk_4.jpg
diff --git a/images/elonmusk_5.jpg b/images/elonmusk_5.jpg
diff --git a/images/elonmusk_6.jpg b/images/elonmusk_6.jpg
diff --git a/images/elonmusk_7.jpg b/images/elonmusk_7.jpg
diff --git a/images/elonmusk_8.jpg b/images/elonmusk_8.jpg
diff --git a/images/elonmusk_9.jpg b/images/elonmusk_9.jpg
diff --git a/images/iphone_0.jpg b/images/iphone_0.jpg
diff --git a/images/iphone_1.jpg b/images/iphone_1.jpg
diff --git a/images/iphone_10.jpg b/images/iphone_10.jpg
diff --git a/images/iphone_11.jpg b/images/iphone_11.jpg
diff --git a/images/iphone_12.jpg b/images/iphone_12.jpg
diff --git a/images/iphone_13.jpg b/images/iphone_13.jpg
diff --git a/images/iphone_14.jpg b/images/iphone_14.jpg
diff --git a/images/iphone_15.jpg b/images/iphone_15.jpg
diff --git a/images/iphone_16.jpg b/images/iphone_16.jpg
diff --git a/images/iphone_17.jpg b/images/iphone_17.jpg
diff --git a/images/iphone_18.jpg b/images/iphone_18.jpg
diff --git a/images/iphone_19.jpg b/images/iphone_19.jpg
diff --git a/images/iphone_2.jpg b/images/iphone_2.jpg
diff --git a/images/iphone_3.jpg b/images/iphone_3.jpg
diff --git a/images/iphone_4.jpg b/images/iphone_4.jpg
diff --git a/images/iphone_5.jpg b/images/iphone_5.jpg
diff --git a/images/iphone_6.jpg b/images/iphone_6.jpg
diff --git a/images/iphone_7.jpg b/images/iphone_7.jpg
diff --git a/images/iphone_8.jpg b/images/iphone_8.jpg
diff --git a/images/iphone_9.jpg b/images/iphone_9.jpg
diff --git a/images/samsung_0.jpg b/images/samsung_0.jpg
diff --git a/images/samsung_1.jpg b/images/samsung_1.jpg
diff --git a/images/samsung_10.jpg b/images/samsung_10.jpg
diff --git a/images/samsung_11.jpg b/images/samsung_11.jpg
diff --git a/images/samsung_12.jpg b/images/samsung_12.jpg
diff --git a/images/samsung_13.jpg b/images/samsung_13.jpg
diff --git a/images/samsung_14.jpg b/images/samsung_14.jpg
diff --git a/images/samsung_15.jpg b/images/samsung_15.jpg
diff --git a/images/samsung_16.jpg b/images/samsung_16.jpg
diff --git a/images/samsung_17.jpg b/images/samsung_17.jpg
diff --git a/images/samsung_18.jpg b/images/samsung_18.jpg
diff --git a/images/samsung_19.jpg b/images/samsung_19.jpg
diff --git a/images/samsung_2.jpg b/images/samsung_2.jpg
diff --git a/images/samsung_3.jpg b/images/samsung_3.jpg
diff --git a/images/samsung_4.jpg b/images/samsung_4.jpg
diff --git a/images/samsung_5.jpg b/images/samsung_5.jpg
diff --git a/images/samsung_6.jpg b/images/samsung_6.jpg
diff --git a/images/samsung_7.jpg b/images/samsung_7.jpg
diff --git a/images/samsung_8.jpg b/images/samsung_8.jpg
diff --git a/images/samsung_9.jpg b/images/samsung_9.jpg
diff --git a/images/virat_0.jpg b/images/virat_0.jpg
diff --git a/images/virat_1.jpg b/images/virat_1.jpg
diff --git a/images/virat_10.jpg b/images/virat_10.jpg
diff --git a/images/virat_11.jpg b/images/virat_11.jpg
diff --git a/images/virat_12.jpg b/images/virat_12.jpg
diff --git a/images/virat_13.jpg b/images/virat_13.jpg
diff --git a/images/virat_14.jpg b/images/virat_14.jpg
diff --git a/images/virat_15.jpg b/images/virat_15.jpg
diff --git a/images/virat_16.jpg b/images/virat_16.jpg
diff --git a/images/virat_17.jpg b/images/virat_17.jpg
diff --git a/images/virat_18.jpg b/images/virat_18.jpg
diff --git a/images/virat_19.jpg b/images/virat_19.jpg
diff --git a/images/virat_2.jpg b/images/virat_2.jpg
diff --git a/images/virat_3.jpg b/images/virat_3.jpg
diff --git a/images/virat_4.jpg b/images/virat_4.jpg
diff --git a/images/virat_5.jpg b/images/virat_5.jpg
diff --git a/images/virat_6.jpg b/images/virat_6.jpg
diff --git a/images/virat_7.jpg b/images/virat_7.jpg
diff --git a/images/virat_8.jpg b/images/virat_8.jpg
diff --git a/images/virat_9.jpg b/images/virat_9.jpg
diff --git a/requirements.txt b/requirements.txt
@@ -0,0 +1,5 @@
+flask
+flask_cors
+requests
+bs4
+pymongo
diff --git a/scrapper.log b/scrapper.log
@@ -0,0 +1,57 @@
+INFO:werkzeug:[31m[1mWARNING: This is a development server. Do not use it in a production deployment. Use a production WSGI server instead.[0m
+ * Running on all addresses (0.0.0.0)
+ * Running on http://127.0.0.1:5000
+ * Running on http://172.18.0.21:5000
+INFO:werkzeug:[33mPress CTRL+C to quit[0m
+INFO:werkzeug:172.18.0.2 - - [21/Feb/2023 23:00:41] "GET / HTTP/1.1" 200 -
+INFO:werkzeug:172.18.0.2 - - [21/Feb/2023 23:00:41] "[36mGET /static/css/style.css HTTP/1.1[0m" 304 -
+INFO:werkzeug:172.18.0.2 - - [21/Feb/2023 23:00:41] "[36mGET /static/css/main.css HTTP/1.1[0m" 304 -
+INFO:root:name 'os' is not defined
+INFO:werkzeug:172.18.0.2 - - [21/Feb/2023 23:00:44] "POST /review HTTP/1.1" 200 -
+INFO:werkzeug:[31m[1mWARNING: This is a development server. Do not use it in a production deployment. Use a production WSGI server instead.[0m
+ * Running on all addresses (0.0.0.0)
+ * Running on http://127.0.0.1:5000
+ * Running on http://172.18.0.21:5000
+INFO:werkzeug:[33mPress CTRL+C to quit[0m
+INFO:root:name 'BeautifulSoup' is not defined
+INFO:werkzeug:172.18.0.2 - - [21/Feb/2023 23:01:18] "POST /review HTTP/1.1" 200 -
+INFO:werkzeug:172.18.0.2 - - [21/Feb/2023 23:01:20] "GET / HTTP/1.1" 200 -
+INFO:werkzeug:172.18.0.2 - - [21/Feb/2023 23:01:20] "[36mGET /static/css/main.css HTTP/1.1[0m" 304 -
+INFO:werkzeug:172.18.0.2 - - [21/Feb/2023 23:01:20] "[36mGET /static/css/style.css HTTP/1.1[0m" 304 -
+INFO:root:name 'BeautifulSoup' is not defined
+INFO:werkzeug:172.18.0.2 - - [21/Feb/2023 23:01:23] "POST /review HTTP/1.1" 200 -
+INFO:root:name 'BeautifulSoup' is not defined
+INFO:werkzeug:172.18.0.2 - - [21/Feb/2023 23:01:23] "POST /review HTTP/1.1" 200 -
+INFO:werkzeug:[31m[1mWARNING: This is a development server. Do not use it in a production deployment. Use a production WSGI server instead.[0m
+ * Running on all addresses (0.0.0.0)
+ * Running on http://127.0.0.1:5000
+ * Running on http://172.18.0.21:5000
+INFO:werkzeug:[33mPress CTRL+C to quit[0m
+INFO:werkzeug:172.18.0.2 - - [21/Feb/2023 23:01:52] "GET / HTTP/1.1" 200 -
+INFO:werkzeug:172.18.0.2 - - [21/Feb/2023 23:01:52] "[36mGET /static/css/main.css HTTP/1.1[0m" 304 -
+INFO:werkzeug:172.18.0.2 - - [21/Feb/2023 23:01:52] "[36mGET /static/css/style.css HTTP/1.1[0m" 304 -
+INFO:werkzeug:172.18.0.2 - - [21/Feb/2023 23:01:52] "POST /review HTTP/1.1" 200 -
+INFO:werkzeug:172.18.0.2 - - [21/Feb/2023 23:01:57] "POST /review HTTP/1.1" 200 -
+INFO:werkzeug:[31m[1mWARNING: This is a development server. Do not use it in a production deployment. Use a production WSGI server instead.[0m
+ * Running on all addresses (0.0.0.0)
+ * Running on http://127.0.0.1:5000
+ * Running on http://172.20.10.9:5000
+INFO:werkzeug:[33mPress CTRL+C to quit[0m
+INFO:werkzeug:172.20.10.9 - - [22/Feb/2023 11:24:04] "GET / HTTP/1.1" 200 -
+INFO:werkzeug:172.20.10.9 - - [22/Feb/2023 11:24:04] "GET /static/css/main.css HTTP/1.1" 200 -
+INFO:werkzeug:172.20.10.9 - - [22/Feb/2023 11:24:04] "GET /static/css/style.css HTTP/1.1" 200 -
+INFO:werkzeug:172.20.10.9 - - [22/Feb/2023 11:24:04] "[33mGET /favicon.ico HTTP/1.1[0m" 404 -
+INFO:werkzeug:172.20.10.9 - - [22/Feb/2023 11:24:16] "POST /review HTTP/1.1" 200 -
+INFO:werkzeug:[31m[1mWARNING: This is a development server. Do not use it in a production deployment. Use a production WSGI server instead.[0m
+ * Running on all addresses (0.0.0.0)
+ * Running on http://127.0.0.1:8000
+ * Running on http://172.18.0.84:8000
+INFO:werkzeug:[33mPress CTRL+C to quit[0m
+INFO:werkzeug:172.18.0.45 - - [27/Jul/2024 16:24:26] "GET / HTTP/1.1" 200 -
+INFO:werkzeug:172.18.0.45 - - [27/Jul/2024 16:24:26] "GET /static/css/main.css HTTP/1.1" 200 -
+INFO:werkzeug:172.18.0.45 - - [27/Jul/2024 16:24:26] "GET /static/css/style.css HTTP/1.1" 200 -
+INFO:werkzeug:172.18.0.45 - - [27/Jul/2024 16:24:27] "[33mGET /favicon.ico HTTP/1.1[0m" 404 -
+INFO:root:bad auth : Authentication failed., full error: {'ok': 0, 'errmsg': 'bad auth : Authentication failed.', 'code': 8000, 'codeName': 'AtlasError'}
+INFO:werkzeug:172.18.0.45 - - [27/Jul/2024 16:24:36] "POST /review HTTP/1.1" 200 -
+INFO:root:bad auth : Authentication failed., full error: {'ok': 0, 'errmsg': 'bad auth : Authentication failed.', 'code': 8000, 'codeName': 'AtlasError'}
+INFO:werkzeug:172.18.0.45 - - [27/Jul/2024 16:25:00] "POST /review HTTP/1.1" 200 -
diff --git a/static/css/main.css b/static/css/main.css
@@ -0,0 +1,35 @@
+body, html {
+    margin: 0;
+    font-family: sans-serif;
+}
+
+.content {
+    margin: 0 auto;
+    width: 400px;
+}
+
+table, td, th {
+    border: 1px solid #aaa;
+}
+
+table {
+    border-collapse: collapse;
+    width: 100%;
+}
+
+th {
+    height: 30px;
+}
+
+td {
+    text-align: center;
+    padding: 5px;
+}
+
+.form {
+    margin-top: 20px;
+}
+
+#content {
+    width: 70%;
+}