Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactor cognizantsoftvision, conarg, continental, crowdstrike and cu… #349

Merged
merged 1 commit into from
May 8, 2024
Merged

Refactor cognizantsoftvision, conarg, continental, crowdstrike and cu… #349

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 32 additions & 28 deletions forRefactor/cognizantsoftvision.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,9 @@
const { Scraper, postApiPeViitor } = require("peviitor_jsscraper");

const generateJob = (job_title, job_link) => ({
job_title,
job_link,
country: "Romania",
county: ["Maramures", "București", "Cluj", "Iași", "Timiș"],
city: ["Baia Mare", "București", "Cluj-Napoca", "Iași", "Timișoara"],
remote: ["Remote", "Hybrid"],
});
const {
Scraper,
postApiPeViitor,
generateJob,
getParams,
} = require("peviitor_jsscraper");

const getJobs = async () => {
const url =
Expand All @@ -22,33 +18,40 @@ const getJobs = async () => {
res.find("script", { type: "application/json" }).text
);

json.props.pageProps.jobOpenings.jobs.forEach((job) => {
if (job.location == "Romania") {
const job_title = job.title;
const job_link = "https://www.cognizantsoftvision.com" + job.link;
jobs.push(generateJob(job_title, job_link));
json.props.pageProps.jobOpenings.jobs.forEach((elem) => {
if (elem.location == "Romania") {
const job_title = elem.title;
const job_link = "https://www.cognizantsoftvision.com" + elem.link;
const cities = [
"Baia Mare",
"Bucuresti",
"Cluj-Napoca",
"Iasi",
"Timisoara",
];
const counties = ["Maramures", "Bucuresti", "Cluj", "Iasi", "Timis"];
const remote = ["Remote", "Hybrid"];

const job = generateJob(
job_title,
job_link,
"Romania",
cities,
counties,
remote
);
jobs.push(job);
}
});

return jobs;
};

const getParams = () => {
const run = async () => {
const company = "CognizantSoftvision";
const logo =
"https://www.cognizantsoftvision.com/react-images/logos/logo-header.png";
const apikey = process.env.APIKEY;
const params = {
company,
logo,
apikey,
};
return params;
};

const run = async () => {
const jobs = await getJobs();
const params = getParams();
const params = getParams(company, logo);
postApiPeViitor(jobs, params);
};

Expand All @@ -57,3 +60,4 @@ if (require.main === module) {
}

module.exports = { run, getJobs, getParams }; // this is needed for our unit test job

41 changes: 15 additions & 26 deletions forRefactor/conarg.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,9 @@
const { Scraper, postApiPeViitor } = require("peviitor_jsscraper");

const generateJob = (job_title, job_link, country, county, city) => ({
job_title,
job_link,
country: "Romania",
county: "Bucuresti",
city: "Bucuresti",
});
const {
Scraper,
postApiPeViitor,
generateJob,
getParams,
} = require("peviitor_jsscraper");

const getJobs = async () => {
const url = "https://www.conarg.co/ro/cariere/oportunitati-de-cariera.html";
Expand All @@ -18,31 +15,23 @@ const getJobs = async () => {
.find("article", { class: "wk-content" })
.findAll("li");

jobsElements.forEach((job) => {
const job_title = job.find("h2").text.trim();
const job_link = "https://www.conarg.co" + job.find("a").attrs.href;

jobs.push(generateJob(job_title, job_link));
jobsElements.forEach((elem) => {
const job_title = elem.find("h2").text.trim();
const job_link = "https://www.conarg.co" + elem.find("a").attrs.href;
const city = "Bucuresti";
const county = "Bucuresti";
const job = generateJob(job_title, job_link, "Romania", city, county);
jobs.push(job);
});

return jobs;
};

const getParams = () => {
const run = async () => {
const company = "Conarg";
const logo = "http://www.conarg.co/images/logo/logo.svg";
const apikey = process.env.APIKEY;
const params = {
company,
logo,
apikey,
};
return params;
};

const run = async () => {
const jobs = await getJobs();
const params = getParams();
const params = getParams(company, logo);
postApiPeViitor(jobs, params);
};

Expand Down
146 changes: 80 additions & 66 deletions forRefactor/continental.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,81 +1,95 @@
"use strict";
const scraper = require("../peviitor_scraper.js");
const { getTownAndCounty } = require("../getTownAndCounty.js");
const { translate_city } = require("../utils.js");
const {
Scraper,
postApiPeViitor,
generateJob,
getParams,
} = require("peviitor_jsscraper");
const { Counties } = require("../getTownAndCounty.js");

const url = "https://jobs.continental.com/en/api/result-list/pagetype-jobs/";
const _counties = new Counties();

const s = new scraper.ApiScraper(url);
const getJobs = async () => {
let url = "https://jobs.continental.com/en/api/result-list/pagetype-jobs/";

let data = {
"tx_conjobs_api[filter][locationSuggestChecksums][]":
"97df691231f6e638f96c63c52454e425",
"tx_conjobs_api[itemsPerPage]": 200,
"tx_conjobs_api[currentPage]": 1,
};
const jobs = [];

const scraper = new Scraper(url);
scraper.config.headers["User-Agent"] = "Mozilla/5.0";

s.headers.headers["Content-Type"] = "application/x-www-form-urlencoded";
const data = {
"tx_conjobs_api[filter][locationSuggestChecksums][]":
"97df691231f6e638f96c63c52454e425",
"tx_conjobs_api[itemsPerPage]": 200,
"tx_conjobs_api[currentPage]": 1,
};

let totalPages;
const formData = new FormData();
for (const key in data) {
formData.append(key, data[key]);
}
const res = await scraper.post(formData);

s.post(data)
.then((d, err) => {
totalPages = d.result.pagination.pagesCount;
})
.then(() => {
let finalJobs = [];
const company = { company: "Continental" };
const totalPages = res.result.pagination.pagesCount;

const fetchData = async () => {
let jobs = [];
for (let i = 1; i <= totalPages; i++) {
data["tx_conjobs_api[currentPage]"] = i;

let fetchData = () => {
return new Promise((resolve) => {
for (let i = 1; i <= totalPages; i++) {
data["tx_conjobs_api[currentPage]"] = i;
s.post(data).then((d) => {
let response = d.result.list;
response.forEach((element) => {
jobs.push(element);
});
if (jobs.length === d.result.numFound) {
resolve(jobs);
}
});
}
});
};

fetchData().then((jobs) => {
jobs.forEach((job) => {
const job_title = job.title;
const job_link = job.absoluteUrl;
const city = job.cityLabel;
const country = job.countryLabel;

const { foudedTown, county } = getTownAndCounty(
translate_city(city.trim().toLowerCase())
);

finalJobs.push({
job_title: job_title,
job_link: job_link,
company: company.company,
city: foudedTown,
county: county,
country: country,
});
const formData = new FormData();
for (const key in data) {
formData.append(key, data[key]);
}

const res = await scraper.post(formData);
res.result.list.forEach((job) => {
jobs.push(job);
});
}
return jobs;
};

console.log(JSON.stringify(finalJobs, null, 2));
scraper.postApiPeViitor(finalJobs, company);
const jobsData = await fetchData();

let logo =
"https://cdn.continental.com/fileadmin/_processed_/3/b/csm_continental_20logo-1920x1080_247d99d89e.png";
await Promise.all(
jobsData.map(async (elem) => {
const job_title = elem.title;
const job_link = "https://jobs.continental.com/en/detail-page/job-detail/" + elem.url;
const city = elem.cityLabel;
const country = elem.countryLabel;

let postLogo = new scraper.ApiScraper(
"https://api.peviitor.ro/v1/logo/add/"
let cities = [];
let counties = [];

const { city: c, county: co } = await _counties.getCounties(
translate_city(city.trim())
);
postLogo.headers.headers["Content-Type"] = "application/json";
postLogo.post(JSON.stringify([{ id: company.company, logo: logo }]));
});
});

if (c) {
cities.push(c);
counties = [...new Set([...counties, ...co])];
}

const job = generateJob(job_title, job_link, country, cities, counties);
jobs.push(job);
})
);

return jobs;
};

const run = async () => {
const company = "Continental";
const logo =
"https://cdn.continental.com/fileadmin/_processed_/3/b/csm_continental_20logo-1920x1080_247d99d89e.png";
const jobs = await getJobs();
const params = getParams(company, logo);
postApiPeViitor(jobs, params);
};

if (require.main === module) {
run();
}

module.exports = { run, getJobs, getParams }; // this is needed for our unit test job
Loading
Loading