-
Notifications
You must be signed in to change notification settings - Fork 2
/
jobfile-hydro-stations.js
81 lines (79 loc) · 2.4 KB
/
jobfile-hydro-stations.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
import _ from 'lodash'
const dbUrl = process.env.DB_URL || 'mongodb://127.0.0.1:27017/hubeau'
export default {
id: 'hubeau-hydro-stations',
store: 'memory',
options: {
workersLimit: 1,
faultTolerant: true,
},
tasks: [{
id: 'stations',
type: 'http',
options: {
url: 'https://hubeau.eaufrance.fr/api/v1/hydrometrie/referentiel/stations?format=geojson&size=10000'
}
}],
hooks: {
tasks: {
after: {
readJson: {},
apply: {
function: (item) => {
let stations = []
if (item.data.features) {
item.data.features.forEach(feature => {
let name = feature.properties.libelle_station || feature.properties.libelle_site || feature.properties.libelle_commune
if (feature.properties.en_service === true) {
let station = _.cloneDeep(feature)
_.set(station, 'properties.name', name)
_.set(station, 'properties.code_station', '#' + feature.properties.code_station) // prefix the code to disable automatic data conversion
stations.push(station)
} else console.log('warning: station ' + name + ' is inactive' )
})
}
console.log('Found ' + stations.length + ' active stations')
item.data = stations
}
},
updateMongoCollection: {
collection: 'hubeau-hydro-stations',
filter: { 'properties.code_station': '<%= properties.code_station %>' },
upsert : true,
chunkSize: 256
},
clearData: {}
}
},
jobs: {
before: {
createStores: { id: 'memory' },
connectMongo: {
url: dbUrl,
// Required so that client is forwarded from job to tasks
clientPath: 'taskTemplate.client'
},
createMongoCollection: {
clientPath: 'taskTemplate.client',
collection: 'hubeau-hydro-stations',
indices: [
[{ 'properties.code_station': 1 }, { unique: true }],
{ geometry: '2dsphere' }
]
}
},
after: {
disconnectMongo: {
clientPath: 'taskTemplate.client'
},
removeStores: [ 'memory' ]
},
error: {
disconnectMongo: {
clientPath: 'taskTemplate.client'
},
removeStores: [ 'memory' ]
}
}
}
}