augementation des ranges
This commit is contained in:
el
parent
9653e55453
commit
7878062edc
9 changed files with 159 additions and 197 deletions
|
|
@ -1,55 +0,0 @@
|
|||
"use strict";
|
||||
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
||||
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
||||
return new (P || (P = Promise))(function (resolve, reject) {
|
||||
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
||||
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
||||
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
||||
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
||||
});
|
||||
};
|
||||
var __importDefault = (this && this.__importDefault) || function (mod) {
|
||||
return (mod && mod.__esModule) ? mod : { "default": mod };
|
||||
};
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.ingestJobOffers = void 0;
|
||||
const client_1 = require("@prisma/client");
|
||||
const FranceTravailService_1 = __importDefault(require("../services/FranceTravailService"));
|
||||
const prisma = new client_1.PrismaClient();
|
||||
const ingestJobOffers = (req, res) => __awaiter(void 0, void 0, void 0, function* () {
|
||||
var _a, _b, _c, _d, _e;
|
||||
try {
|
||||
const jobOffers = yield FranceTravailService_1.default.getJobOffers({ range: '0-149' });
|
||||
for (const offre of jobOffers.resultats) {
|
||||
const mappedOffer = {
|
||||
id: offre.id,
|
||||
title: offre.intitule,
|
||||
description: offre.description,
|
||||
publicationDate: new Date(offre.dateCreation),
|
||||
romeCode: offre.romeCode,
|
||||
romeLabel: offre.romeLibelle,
|
||||
locationLabel: ((_a = offre.lieuTravail) === null || _a === void 0 ? void 0 : _a.libelle) || null,
|
||||
postalCode: ((_b = offre.lieuTravail) === null || _b === void 0 ? void 0 : _b.codePostal) || null,
|
||||
departmentCode: ((_c = offre.lieuTravail) === null || _c === void 0 ? void 0 : _c.codeDepartement) || null,
|
||||
cityName: ((_d = offre.lieuTravail) === null || _d === void 0 ? void 0 : _d.ville) || null,
|
||||
companyName: ((_e = offre.entreprise) === null || _e === void 0 ? void 0 : _e.nom) || null,
|
||||
contractType: offre.typeContrat,
|
||||
contractLabel: offre.libelleTypeContrat,
|
||||
};
|
||||
yield prisma.jobOffer.upsert({
|
||||
where: { id: mappedOffer.id },
|
||||
update: mappedOffer,
|
||||
create: mappedOffer,
|
||||
});
|
||||
}
|
||||
res.status(200).json({
|
||||
message: 'Job offers ingested successfully',
|
||||
count: jobOffers.resultats.length,
|
||||
});
|
||||
}
|
||||
catch (error) {
|
||||
console.error('Error ingesting job offers:', error);
|
||||
res.status(500).json({ error: 'Failed to ingest job offers' });
|
||||
}
|
||||
});
|
||||
exports.ingestJobOffers = ingestJobOffers;
|
||||
|
|
@ -4,40 +4,89 @@ import FranceTravailService from '../services/FranceTravailService';
|
|||
|
||||
const prisma = new PrismaClient();
|
||||
|
||||
// Fonction utilitaire pour introduire un délai
|
||||
const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
|
||||
|
||||
export const ingestJobOffers = async (req: Request, res: Response) => {
|
||||
console.log('Ingesting job offers...'); // Log pour vérifier que le fichier TS est utilisé
|
||||
|
||||
const BATCH_SIZE = 149; // Limite par requête, -1 pour une plage 0-149
|
||||
const TOTAL_OFFERS_TO_FETCH = 1000; // Nombre total d'offres que vous voulez essayer d'ingérer
|
||||
const DELAY_MS = 150; // Délai en ms entre chaque requête pour respecter le rate limit (1000ms/10appels = 100ms, donc 150ms est prudent)
|
||||
|
||||
let totalIngestedCount = 0;
|
||||
let page = 0;
|
||||
|
||||
try {
|
||||
const jobOffers = await FranceTravailService.getJobOffers({ range: '0-149' });
|
||||
for (let start = 0; start < TOTAL_OFFERS_TO_FETCH; start += (BATCH_SIZE + 1)) {
|
||||
const end = start + BATCH_SIZE;
|
||||
const range = `${start}-${end}`;
|
||||
console.log(`Workspaceing offers with range: ${range}`);
|
||||
|
||||
for (const offre of jobOffers.resultats) {
|
||||
const mappedOffer = {
|
||||
id: offre.id,
|
||||
title: offre.intitule,
|
||||
description: offre.description,
|
||||
publicationDate: new Date(offre.dateCreation),
|
||||
romeCode: offre.romeCode,
|
||||
romeLabel: offre.romeLibelle,
|
||||
locationLabel: offre.lieuTravail?.libelle || null,
|
||||
postalCode: offre.lieuTravail?.codePostal || null,
|
||||
departmentCode: offre.lieuTravail?.codeDepartement || null,
|
||||
cityName: offre.lieuTravail?.ville || null,
|
||||
companyName: offre.entreprise?.nom || null,
|
||||
contractType: offre.typeContrat,
|
||||
contractLabel: offre.libelleTypeContrat,
|
||||
};
|
||||
try {
|
||||
const jobOffersResponse = await FranceTravailService.getJobOffers({ range });
|
||||
const jobOffers = jobOffersResponse.resultats; // Assurez-vous que la réponse contient bien un tableau 'resultats'
|
||||
|
||||
await prisma.jobOffer.upsert({
|
||||
where: { id: mappedOffer.id },
|
||||
update: mappedOffer,
|
||||
create: mappedOffer,
|
||||
});
|
||||
if (!jobOffers || jobOffers.length === 0) {
|
||||
console.log(`No more offers found for range ${range}. Stopping ingestion.`);
|
||||
break; // Arrête la boucle s'il n'y a plus d'offres
|
||||
}
|
||||
|
||||
for (const offre of jobOffers) {
|
||||
const mappedOffer = {
|
||||
id: String(offre.id), // S'assurer que l'ID est une string pour Prisma
|
||||
title: offre.intitule || 'N/A', // Ajouter des valeurs par défaut pour les champs obligatoires si l'API peut retourner null
|
||||
description: offre.description || 'N/A',
|
||||
publicationDate: new Date(offre.dateCreation),
|
||||
romeCode: offre.romeCode || null,
|
||||
romeLabel: offre.romeLibelle || null,
|
||||
locationLabel: offre.lieuTravail?.libelle || null,
|
||||
postalCode: offre.lieuTravail?.codePostal || null,
|
||||
departmentCode: offre.lieuTravail?.codeDepartement || null,
|
||||
cityName: offre.lieuTravail?.ville || null,
|
||||
companyName: offre.entreprise?.nom || null,
|
||||
contractType: offre.typeContrat || null,
|
||||
contractLabel: offre.libelleTypeContrat || null,
|
||||
};
|
||||
|
||||
// Assurer la validité des dates pour Prisma
|
||||
if (isNaN(mappedOffer.publicationDate.getTime())) {
|
||||
console.warn(`Invalid dateCreation for offer ID: ${offre.id}. Using current date.`);
|
||||
mappedOffer.publicationDate = new Date();
|
||||
}
|
||||
|
||||
await prisma.jobOffer.upsert({
|
||||
where: { id: mappedOffer.id },
|
||||
update: mappedOffer,
|
||||
create: mappedOffer,
|
||||
});
|
||||
totalIngestedCount++;
|
||||
}
|
||||
|
||||
console.log(`Ingested ${jobOffers.length} offers for range ${range}. Total: ${totalIngestedCount}`);
|
||||
|
||||
} catch (batchError) {
|
||||
// Gérer les erreurs spécifiques à un appel de lot (ex: rate limit temporaire)
|
||||
const axiosError = batchError as any; // Utiliser 'any' pour un accès flexible à la propriété 'response'
|
||||
console.warn(`Error fetching batch for range ${range}:`, axiosError.response?.data || axiosError.message);
|
||||
// Si c'est une erreur de rate limit, vous pourriez vouloir augmenter le délai ou réessayer
|
||||
}
|
||||
|
||||
// Introduire un délai avant la prochaine requête pour respecter le rate limit
|
||||
if (start + (BATCH_SIZE + 1) < TOTAL_OFFERS_TO_FETCH) {
|
||||
await sleep(DELAY_MS);
|
||||
}
|
||||
}
|
||||
|
||||
res.status(200).json({
|
||||
message: 'Job offers ingested successfully',
|
||||
count: jobOffers.resultats.length,
|
||||
message: `Job offers ingestion complete. Total offers processed: ${totalIngestedCount}`,
|
||||
count: totalIngestedCount,
|
||||
});
|
||||
|
||||
} catch (error) {
|
||||
console.error('Error ingesting job offers:', error);
|
||||
console.error('Fatal error during job offers ingestion process:', error);
|
||||
res.status(500).json({ error: 'Failed to ingest job offers' });
|
||||
} finally {
|
||||
await prisma.$disconnect(); // Déconnecter Prisma après l'opération
|
||||
}
|
||||
};
|
||||
};
|
||||
|
|
@ -1,78 +0,0 @@
|
|||
"use strict";
|
||||
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
||||
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
||||
return new (P || (P = Promise))(function (resolve, reject) {
|
||||
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
||||
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
||||
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
||||
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
||||
});
|
||||
};
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.searchLocalJobOffers = void 0;
|
||||
const client_1 = require("@prisma/client");
|
||||
const prisma = new client_1.PrismaClient();
|
||||
const searchLocalJobOffers = (req, res) => __awaiter(void 0, void 0, void 0, function* () {
|
||||
try {
|
||||
const page = parseInt(req.query.page) || 1;
|
||||
const limit = parseInt(req.query.limit) || 10;
|
||||
const skip = (page - 1) * limit;
|
||||
const take = limit;
|
||||
const sortBy = req.query.sortBy || 'publicationDate';
|
||||
const sortOrder = req.query.sortOrder || 'desc';
|
||||
const keyword = req.query.keyword;
|
||||
const location = req.query.location;
|
||||
const contractType = req.query.contractType;
|
||||
const where = {};
|
||||
if (keyword) {
|
||||
where.OR = [
|
||||
{ title: { contains: keyword, mode: 'insensitive' } },
|
||||
{ description: { contains: keyword, mode: 'insensitive' } },
|
||||
];
|
||||
}
|
||||
if (location) {
|
||||
where.AND = [
|
||||
...(where.AND || []),
|
||||
{
|
||||
OR: [
|
||||
{ locationLabel: { contains: location, mode: 'insensitive' } },
|
||||
{ postalCode: { contains: location, mode: 'insensitive' } },
|
||||
{ cityName: { contains: location, mode: 'insensitive' } },
|
||||
{ departmentCode: { contains: location, mode: 'insensitive' } },
|
||||
],
|
||||
},
|
||||
];
|
||||
}
|
||||
if (contractType) {
|
||||
where.AND = [
|
||||
...(where.AND || []),
|
||||
{ contractType: contractType },
|
||||
];
|
||||
}
|
||||
const orderBy = {};
|
||||
if (sortBy) {
|
||||
orderBy[sortBy] = sortOrder === 'asc' ? 'asc' : 'desc';
|
||||
}
|
||||
else {
|
||||
orderBy.publicationDate = 'desc'; // Tri par défaut
|
||||
}
|
||||
const jobs = yield prisma.jobOffer.findMany({
|
||||
skip,
|
||||
take,
|
||||
where,
|
||||
orderBy,
|
||||
});
|
||||
const total = yield prisma.jobOffer.count({ where });
|
||||
res.status(200).json({
|
||||
jobs,
|
||||
total,
|
||||
page,
|
||||
limit,
|
||||
});
|
||||
}
|
||||
catch (error) {
|
||||
console.error('Error searching job offers:', error);
|
||||
res.status(500).json({ error: 'Failed to search job offers' });
|
||||
}
|
||||
});
|
||||
exports.searchLocalJobOffers = searchLocalJobOffers;
|
||||
Loading…
Add table
Add a link
Reference in a new issue