wind-js-server/app.js

var express = require("express");
var moment = require("moment");
var http = require('http');
var request = require('postman-request');
var fs = require('fs');
var Q = require('q');
var cors = require('cors');
const iconv = require('iconv-lite');
const { upload, uploadfileFn } = require("./src/tifupload");

var app = express();
var port = process.env.PORT || 7000;
var baseDir = 'http://nomads.ncep.noaa.gov/cgi-bin/filter_gfs_1p00.pl';

// cors config
var whitelist = [
	'http://localhost:3000',
	'http://localhost:5000',
];

var corsOptions = {
	origin: function (origin, callback) {
		var originIsWhitelisted = whitelist.indexOf(origin) !== -1;
		callback(null, originIsWhitelisted);
	}
};

app.listen(port, function (err) {
	console.log(" running server on port " + port);
});
app.use('/public', express.static('./public'));
app.get('/', cors(corsOptions), function (req, res) {
	res.send('hello wind-server.. go to /latest for wind data..');
});

app.post("/file/upload", upload.single('file'), uploadfileFn)

app.get('/alive', cors(corsOptions), function (req, res) {
	res.send('wind-server is alive');
});
const LevelArray = ["lev_10_m_above_ground", 'lev_1000_mb', 'lev_100_mb', 'lev_250_mb', 'lev_500_mb', "lev_750_mb"];
app.get('/latest', cors(corsOptions), function (req, res) {
	let { leavel } = req.query;
	if (!LevelArray.includes(leavel)) {
		res.send({
			code: 400,
			message: `参数需要为${LevelArray.join(',')}其一`
		})
		return
	}
	if (!leavel) {
		leavel = LevelArray[0]
	}

	/**
	 * Find and return the latest available 6 hourly pre-parsed JSON data
	 *
	 * @param targetMoment {Object} UTC moment
	 */
	function sendLatest(targetMoment, repeatTime) {
		var stamp = moment(targetMoment).format('YYYYMMDD') + '/' + roundHours(moment(targetMoment).hour(), 6) + '/atmos';
		var fileName = __dirname + "/json-data/" + stamp.replaceAll('/', '_').replace('/a', '\\a') + leavel + ".json";

		if (responseSent) {
			return;
		}

		res.setHeader('Access-Control-Allow-Origin', '*');
		res.setHeader('Content-Type', 'application/json');
		res.sendFile(fileName, {}, function (err) {
			if (err) {
				console.log(fileName + ' doesnt exist yet, trying previous interval..');
				if (!repeatTime) repeatTime = 0
				sendLatest(moment(targetMoment).subtract(6, 'hours'), repeatTime++);
				return
			} else {
				responseSent = true;
			}
		});
		return
	}

	let responseSent = false;
	sendLatest(moment().utc());
});

app.get('/nearest', cors(corsOptions), function (req, res, next) {

	var time = req.query.timeIso;
	var limit = req.query.searchLimit;
	var searchForwards = false;

	/**
	 * Find and return the nearest available 6 hourly pre-parsed JSON data
	 * If limit provided, searches backwards to limit, then forwards to limit before failing.
	 *
	 * @param targetMoment {Object} UTC moment
	 */
	function sendNearestTo(targetMoment) {

		if (limit && Math.abs(moment.utc(time).diff(targetMoment, 'days')) >= limit) {
			if (!searchForwards) {
				searchForwards = true;
				sendNearestTo(moment(targetMoment).add(limit, 'days'));
				return;
			}
			else {
				return next(new Error('No data within searchLimit'));
			}
		}

		var stamp = moment(targetMoment).format('YYYYMMDD') + '/' + roundHours(moment(targetMoment).hour(), 6) + '/atmos';
		var fileName = __dirname + "/json-data/" + stamp.replaceAll('/', '_').replace('/a', '\\a') + ".json";

		res.setHeader('Content-Type', 'application/json');
		res.sendFile(fileName, {}, function (err) {
			if (err) {
				var nextTarget = searchForwards ? moment(targetMoment).add(6, 'hours') : moment(targetMoment).subtract(6, 'hours');
				sendNearestTo(nextTarget);
			}
		});
	}

	if (time && moment(time).isValid()) {
		sendNearestTo(moment.utc(time));
	}
	else {
		return next(new Error('Invalid params, expecting: timeIso=ISO_TIME_STRING'));
	}

});

/**
 *
 * Ping for new data every 3 hours
 *
 */
setInterval(function () {

	run(moment.utc());

}, 3 * 60 * 60000);

/**
 *
 * @param targetMoment {Object} moment to check for new data
 */
function run(targetMoment) {

	// const LevelArray = ['lev_750_mb'];
	for (const item of LevelArray) {
		getGribData(targetMoment, item).then(function (response) {
			if (response.stamp) {
				convertGribToJson(response.stamp, response.targetMoment, response.level);
			}
		});
	}
}

/**
 *
 * Finds and returns the latest 6 hourly GRIB2 data from NOAAA
 *
 * @returns {*|promise}
 */
function getGribData(targetMoment, item) {

	var deferred = Q.defer();

	function runQuery(targetMoment, item) {
		// only go 1 day deep
		if (moment.utc().diff(targetMoment, 'days') > 2) {
			console.log('超出限定，不再向下查询', item);
			return;
		}
		var stamp = moment(targetMoment).format('YYYYMMDD') + '/' + roundHours(moment(targetMoment).hour(), 6) + '/atmos';
		if (checkPath('json-data/' + stamp.replaceAll('/', '_') + item + '.json')) {//避免重复请求覆盖
			runQuery(moment(targetMoment).subtract(6, 'hours'), item);
			return
		}
		const param = {
			file: 'gfs.t' + roundHours(moment(targetMoment).hour(), 6) + 'z.pgrb2.1p00.f000',
			var_TMP: 'on',
			var_UGRD: 'on',
			var_VGRD: 'on',
			leftlon: 0,
			rightlon: 360,
			toplat: 90,
			bottomlat: -90,
			dir: '/gfs.' + stamp
		}
		param[item] = 'on';
		request.get({
			url: baseDir,
			qs: param
			//https://nomads.ncep.noaa.gov/cgi-bin/filter_gfs_1p00.pl?file = gfs.t00z.pgrb2.1p00.f000&lev_800_mb=on&leftlon=0 &rightlon=360 & toplat=90 & bottomlat=-90 & dir=/gfs.20230302/00/atmos
		}).on('error', function (err) {
			runQuery(moment(targetMoment).subtract(6, 'hours'), item);
		}).on('response', function (response) {
			console.log('response ' + response.statusCode + ' | ' + stamp);
			if (response.statusCode != 200) {
				if (response.statusCode == 404) {
					runQuery(moment(targetMoment).subtract(6, 'hours'), item);
				} else {
					deferred.resolve({ stamp: false, targetMoment: false, level: item });
				}
			} else {
				// don't rewrite stamps
				if (!checkPath('json-data/' + stamp.replaceAll('/', '_') + item + '.json', false)) {

					console.log('转换' + stamp.replaceAll('/', '_'), item);

					// mk sure we've got somewhere to put output
					checkPath('grib-data', true);

					// pipe the file, resolve the valid time stamp
					var file = fs.createWriteStream("grib-data/" + stamp.replaceAll('/', '_') + item + ".f000");
					response.pipe(file);
					file.on('finish', function () {
						file.close();
						deferred.resolve({ stamp: stamp.replaceAll('/', '_'), targetMoment: targetMoment, level: item });
					});
				}
				else {
					console.log('already have ' + stamp.replaceAll('/', '_') + ', not looking further');
					deferred.resolve({ stamp: false, targetMoment: false, level: item });
				}
			}
		});
	}



	runQuery(targetMoment, item);
	return deferred.promise;
}

function convertGribToJson(stamp, targetMoment, level) {
	// debugger
	// mk sure we've got somewhere to put output
	checkPath('json-data', true);

	var exec = require('child_process').exec, child;

	var commandStr = `grib2json --data --output ./json-data/${stamp.replaceAll('/', '_').replace('/a', '\\a') + level}.json --names --compact ./grib-data/${stamp.replaceAll('/', '_').replace('/a', '\\a') + level}.f000`
	child = exec(commandStr,
		{ maxBuffer: 500 * 1024, encoding: 'buffer', },
		function (error, stdout, stderr) {
			if (error) {
				console.log('exec error: ', error, iconv.decode(stderr, 'cp936'))
			} else {
				console.log("转换中..");
				// don't keep raw grib data
				exec('rm grib-data/*');

				// if we don't have older stamp, try and harvest one
				var prevMoment = moment(targetMoment).subtract(6, 'hours');
				var prevStamp = prevMoment.format('YYYYMMDD') + '/' + roundHours(prevMoment.hour(), 6) + '/atmos';

				if (!checkPath('json-data/' + prevStamp.replaceAll('/', '_').replace('/a', '\\a') + level + '.json', false)) {
					console.log("attempting to harvest older data " + stamp);
					run(prevMoment);
				} else {
					console.log('got older, no need to harvest further' + stamp);
				}
			}
		});
}

/**
 *
 * Round hours to expected interval, e.g. we're currently using 6 hourly interval
 *  00 || 06 || 12 || 18
 *
 * @param hours
 * @param interval
 * @returns {String}
 */
function roundHours(hours, interval) {
	if (interval > 0) {
		var result = (Math.floor(hours / interval) * interval);
		return result < 10 ? '0' + result.toString() : result;
	}
}

/**
 * Sync check if path or file exists
 *
 * @param path {string}
 * @param mkdir {boolean} create dir if doesn't exist
 * @returns {boolean}
 */
function checkPath(path, mkdir) {
	try {
		fs.statSync(path);
		return true;
	} catch (e) {
		if (mkdir) {
			fs.mkdirSync(path);
		}
		return false;
	}
}

// init harvest
run(moment.utc());