Commit 12f456f6 authored by Xuxiaodong's avatar Xuxiaodong

添pentad,monthly入库定时任务(未开启)

parent e1f9dbba
package com.jianju.di.Manager;
import java.io.ByteArrayOutputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
public class Model {
public float minlon;
public float maxlon;
public float minlat;
public float maxlat;
public String dataCode;
public String type;
public ArrayList<byte[]> arrayListbyte=new ArrayList<byte[]>();
public ArrayList<java.sql.Date> arrayListsqlDate=new ArrayList<java.sql.Date>();;
public ArrayList<Integer> arrayListTimeInt=new ArrayList<Integer>();
public ArrayList<Integer> getArrayListTimeInt()
{
return arrayListTimeInt;
}
public ArrayList<java.sql.Date> getArrayListsqlDate() {
return arrayListsqlDate;
}
public ArrayList<byte[]> getArrayListbyte() {
return arrayListbyte;
}
public String getDataCode()
{
return dataCode;
}
public String getType()
{
return type;
}
public float getMinlon()
{
return minlon;
}
public float getMinlat()
{
return minlat;
}
public float getMaxlat()
{
return maxlat;
}
public float getMaxlon()
{
return maxlon;
}
public byte[] floatArrayToByteArray(float[] data) {
ByteArrayOutputStream out = new ByteArrayOutputStream();
DataOutputStream dataOutputStream = new DataOutputStream(out);
for (int i = 0; i < data.length; i++) {
try {
dataOutputStream.writeFloat(data[i]);
} catch (IOException e) {
e.printStackTrace();
}
}
return out.toByteArray();
}
public byte[] floatArray2DToByteArray(float[][]data,int length,int wight) {
ByteArrayOutputStream out = new ByteArrayOutputStream();
DataOutputStream dataOutputStream = new DataOutputStream(out);
for (int i = 0; i < length; i++) {
for(int j=0;j<wight;j++) {
try {
dataOutputStream.writeFloat(data[i][j]);
} catch (IOException e) {
e.printStackTrace();
}
}
}
return out.toByteArray();
}
public Date plusDay(int number) throws ParseException {
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); // 日期格式
Date date = dateFormat.parse("1800-01-01"); // 指定日期
Date newDate = addDate(date, number); // 指定日期加上20天
return newDate;
}
public Date addDate(Date date,long day) throws ParseException {
long time = date.getTime(); // 得到指定日期的毫秒数
day = day*24*60*60*1000; // 要加上的天数转换成毫秒数
time+=day; // 相加得到新的毫秒数
return new Date(time); // 将毫秒数转换成日期
}
public Date plusDayDERF2(int number) throws ParseException {
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); // 日期格式
Date date = dateFormat.parse("2019-04-10"); // 指定日期
Date newDate = addDate(date, number); // 指定日期加上20天
return newDate;
}
public int CountH(int day)
{
if(day>=1&&day<=5)
{
return 1;
}
if(day>5&&day<=10)
{
return 2;
}
if(day>10&&day<=15)
{
return 3;
}
if(day>15&&day<=20)
{
return 4;
}
if(day>20&&day<=25)
{
return 5;
}
if(day>25&&day<=31)
{
return 6;
}
return 0;
}
}
package com.jianju.di.Manager;
//import DataBaseManager.*;
import visad.browser.Convert;
import java.io.*;
......@@ -24,6 +24,35 @@ public class PostGISMannager {
m_tablename=tablename;
}
public void createOISSTV2(String tableName) throws SQLException
{
String sql="create table if not exists public."+tableName+
"(byteData bytea,"+
"time date,"+
"minlon numeric,"+
"maxlon numeric,"+
"maxlat numeric,"+
"minlat numeric,"+
"type char(50),"+
"dataCode char(50),"+
"geom geometry);";
stmt.executeUpdate(sql);
}
public void createSSTV4(String tableName) throws SQLException
{
String sql="create table if not exists public."+tableName+
"(byteData bytea,"+
"time integer,"+
"minlon numeric,"+
"maxlon numeric,"+
"maxlat numeric,"+
"minlat numeric,"+
"type char(50),"+
"dataCode char(50),"+
"geom geometry);";
stmt.executeUpdate(sql);
}
public void createLandPrecip(String tableName) throws SQLException
{
String sql="create table if not exists public."+tableName+
......@@ -211,6 +240,17 @@ public class PostGISMannager {
// return status;
// }
public int selectMaxTime(String tableName) throws SQLException {
ResultSet rs = stmt.executeQuery("select * from "+tableName);
int time=0;
while(rs.next()){
time = rs.getInt("time");
}
return time;
}
// public int insertOisstv2MonToPG(String tableName,String dataPath,boolean ice) throws ParseException, IOException {
// int status=0;
// String geom;
......@@ -275,7 +315,7 @@ public class PostGISMannager {
String type2="sst";
String type3="u_flx";
String datacode="REAN_GODAS_MUL_PEN";
String datacode="REAN_GODAS_MUL_MON";
String time;
float minLat,maxLat,minLon,maxLon;
String mongoID = null;
......@@ -513,6 +553,127 @@ public class PostGISMannager {
}
// public int insertSSTV4(String tableName,String dataPath) throws ParseException, IOException, SQLException {
// int status=0;
// String geom;
// float minLat,maxLat,minLon,maxLon;
//
// SSTV4 readSST=new SSTV4();
// float[][][] priecipData=readSST.readData(dataPath);
// String type=readSST.getType();
// String datacode=readSST.getDataCode();
//
//
// int oldtime=selectMaxTime("sstv4");
//
// minLon=readSST.getMinlon();
// maxLat=readSST.getMaxlat();
// maxLon=readSST.getMaxlon();
// minLat=readSST.getMinlat();
// ArrayList<Integer> arrayListsqlDate=readSST.getArrayListTimeInt();
// ArrayList<byte[]> arrayListbyte=readSST.getArrayListbyte();
// geom="POLYGON(("+minLon+" "+maxLat+","+maxLon+" "+maxLat+","+maxLon+" "+minLat+","+minLon+" "+minLat+","+minLon+" "+maxLat+"))";
//
//
// for(int i=0;i<arrayListsqlDate.size();i++)
// {
// PreparedStatement ps = null;
// ResultSet st = null;
// try
// {
// int sqlDate = arrayListsqlDate.get(i);
// if(sqlDate>oldtime) {
// System.out.println("success connect");
// String sql = "insert into " + tableName + "(bytedata,time,minlon,maxlon,maxlat,minlat,type,dataCode,geom)" +
// " values(?,?,?,?,?,?,?,?," + "'" + geom + "'" + ")";
// try {
// ps = c.prepareStatement(sql);
// } catch (SQLException e) {
// e.printStackTrace();
// }
// float[][] tempData = priecipData[i];
// byte[] byteData = arrayListbyte.get(i);
// ps.setBytes(1, byteData);
// ps.setInt(2, sqlDate);
// ps.setFloat(3, minLon);
// ps.setFloat(4, maxLon);
// ps.setFloat(5, maxLat);
// ps.setFloat(6, minLat);
// ps.setString(7, type);
// ps.setString(8, datacode);
// status = ps.executeUpdate();
// }
//
// }
// catch (Exception ex)
// {
// ex.printStackTrace(System.out);
// }
//
// }
//
// return status;
// }
public int insertOISSTV2(String tableName,String dataPath) throws ParseException, IOException, SQLException {
int status=0;
String geom;
float minLat,maxLat,minLon,maxLon;
ReadOisstv2Highres oisst=new ReadOisstv2Highres();
oisst.readDataFromOisstv2Highres("sst.day.mean.1981.v2.nc");
String type=oisst.getType();
String datacode=oisst.getDataCode();
minLon=oisst.getMinlon();
maxLat=oisst.getMaxlat();
maxLon=oisst.getMaxlon();
minLat=oisst.getMinlat();
ArrayList<Date> arrayListsqlDate=oisst.getArrayListsqlDate();
ArrayList<byte[]> arrayListbyte=oisst.getArrayListbyte();
geom="POLYGON(("+minLon+" "+maxLat+","+maxLon+" "+maxLat+","+maxLon+" "+minLat+","+minLon+" "+minLat+","+minLon+" "+maxLat+"))";
for(int i=0;i<arrayListsqlDate.size();i++)
{
PreparedStatement ps = null;
ResultSet st = null;
try
{
Date sqlDate = arrayListsqlDate.get(i);
System.out.println("success connect");
String sql = "insert into " + tableName + "(bytedata,time,minlon,maxlon,maxlat,minlat,type,dataCode,geom)" +
" values(?,?,?,?,?,?,?,?," + "'" + geom + "'" + ")";
try {
ps = c.prepareStatement(sql);
} catch (SQLException e) {
e.printStackTrace();
}
byte[] byteData = arrayListbyte.get(i);
ps.setBytes(1, byteData);
ps.setDate(2, sqlDate);
ps.setFloat(3, minLon);
ps.setFloat(4, maxLon);
ps.setFloat(5, maxLat);
ps.setFloat(6, minLat);
ps.setString(7, type);
ps.setString(8, datacode);
status = ps.executeUpdate();
}
catch (Exception ex)
{
ex.printStackTrace(System.out);
}
}
return status;
}
private static byte[] floatArray2DToByteArray(float[][]data,int length,int wight) {
ByteArrayOutputStream out = new ByteArrayOutputStream();
DataOutputStream dataOutputStream = new DataOutputStream(out);
......
package com.jianju.di.Manager;
import ucar.ma2.Array;
import ucar.nc2.NetcdfFile;
import java.io.*;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
public class ReadOisstv2Highres extends Model {
public ReadOisstv2Highres()
{
}
public float[][][] readDataFromOisstv2Highres(String files) throws IOException, ParseException {
type="sst";
dataCode="REAN_NCEP_OISST_MUL_DAY";
NetcdfFile ncFile = NetcdfFile.open(files);
ucar.nc2.Variable v1 = ncFile.findVariable("lat");
Array latdata=v1.read();
float[] flat=(float[]) latdata.copyTo1DJavaArray();
maxlat=flat[flat.length-1];
minlat=flat[0];
ucar.nc2.Variable v2 = ncFile.findVariable("lon");
Array londata=v2.read();
float[] flon=(float[]) londata.copyTo1DJavaArray();
maxlon=flon[flon.length-1];
minlon=flon[0];
ucar.nc2.Variable v4 = ncFile.findVariable("time");
Array Vartime=v4.read();
double[] ArryTime=(double[]) Vartime.copyTo1DJavaArray();
Date[] arryDaya=new Date[ArryTime.length];
for(int i=0;i<ArryTime.length;i++)
{
arryDaya[i]=plusDay((int)ArryTime[i]);
java.sql.Date sqlDate=new java.sql.Date(arryDaya[i].getTime());
arrayListsqlDate.add(sqlDate);
}
ucar.nc2.Variable v3 = ncFile.findVariable("sst");
Array data=v3.read();
float[][][] javaArray = (float [][][]) data.copyToNDJavaArray();
//
for(int i=0;i<ArryTime.length;i++)
{
//float[][] temp=
byte[] tempbyte=floatArray2DToByteArray(javaArray[i],flat.length,flon.length);
arrayListbyte.add(tempbyte);
}
return javaArray;
}
// public float[][][] readDataFromsstv2_anom(String files,String elements,String lat,String lon,String time) throws IOException, ParseException {
// NetcdfFile ncFile = NetcdfFile.open(files);
// System.out.println("opne Success");
// ucar.nc2.Variable v = ncFile.findVariable(elements);
// int[] b=v.getShape();
//
// ucar.nc2.Variable v1 = ncFile.findVariable(lat);
// Array latdata=v1.read();
// ucar.nc2.Variable v2 = ncFile.findVariable(lon);
// Array londata=v2.read();
//
// ucar.nc2.Variable v4 = ncFile.findVariable(time);
// Array Vartime=v4.read();
// double[] ArryTime=(double[]) Vartime.copyTo1DJavaArray();
// Date[] arryDaya=new Date[ArryTime.length];
// for(int i=0;i<ArryTime.length;i++)
// {
// arryDaya[i]=plusDay((int)ArryTime[i]);
// }
// System.out.println(ArryTime[121]);
// ucar.nc2.Variable v3 = ncFile.findVariable(elements);
// Array data=v3.read();
// float[][][] javaArray = (float [][][]) data.copyToNDJavaArray();
//
// return javaArray;
//
// }
}
......@@ -14,16 +14,42 @@ public class Test {
public static void main(String[] arg) throws ClassNotFoundException, SQLException, IOException, SQLException, IOException, InvalidRangeException, ParserConfigurationException, SAXException, TransformerException, ParseException {
// 数据入库Test
//数据入库Test
// MongodbManager m_MyMongoDB = new MongodbManager("10.40.16.39", 27088, "Data", "day");
// PostGISMannager postGIS = new PostGISMannager("localhost", "5432",
// "postgis_25_sample", "GODAS", "postgres", "123456");
// postGIS.createTableGDAS("GODAS");
// postGIS.insertGodasToPG("GODAS", "E:\\NewWeather\\godas.P.20200105.grb");
//
// PostGISMannager postGIS = new PostGISMannager("localhost", "5432",
// "postgis_25_sample", "GODAS", "postgres", "123456");
// postGIS.createOlrDay("Olr");
// postGIS.insertOlrToPG("olr","E:\\NOAA\\#20050718");
// PostGISMannager postGIS = new PostGISMannager("localhost", "5432",
// "postgis_25_sample", "GODAS", "postgres", "123456");
// postGIS.createOisstv2Mon("oisst");
// postGIS.insertOisstv2MonToPG("oisst","E:\\NewWeather\\oiv2mon.198111",false);
// PostGISMannager postGIS = new PostGISMannager("localhost", "5432",
// "postgis_25_sample", "CMAP", "postgres", "123456");
// postGIS.createCMAP("CMAP");
// postGIS.insertCMAPToPG("CMAP","E:\\NewWeather\\precip.mon.mean.nc");
// PostGISMannager postGIS = new PostGISMannager("localhost", "5432",
// "postgis_25_sample", "CMAP", "postgres", "123456");
// postGIS.createLandPrecip("LandPrecip");
// postGIS.insertLandPrecipToPG("LandPrecip","E:\\NewWeather\\precip.mon.mean.1x1.nc");
// PostGISMannager postGIS = new PostGISMannager("localhost", "5432",
//// "postgis_25_sample", "CMAP", "postgres", "123456");
//// postGIS.createSSTV4("SSTV4");
//// postGIS.insertSSTV4("SSTV4","E:\\NewWeather\\sst.mnmean.v4.nc");
PostGISMannager postGIS = new PostGISMannager("localhost", "5432",
"postgis_25_sample", "oisstv2", "postgres", "123456");
postGIS.createOISSTV2("oisstv2");
postGIS.insertOISSTV2("oisstv2","sst.day.mean.1981.v2.nc");
}
}
......@@ -40,7 +40,7 @@ public class Di {
Date PROCESS_END_TIME;
String FILE_NAME_O;
String FILE_NAME_N;
String FILE_SIZE;
long FILE_SIZE;
String PROCESS_STATE;
String BUSINESS_STATE;
Date RECORD_TIME;
......@@ -71,7 +71,7 @@ public class Di {
fields.setPROCESS_END_TIME(new Date()); //入库完成时间 或者失败的时间
fields.setFILE_NAME_O("#20200321");
fields.setFILE_NAME_N("#20200321");
fields.setFILE_SIZE("43"); //单位B
fields.setFILE_SIZE(43); //单位B
fields.setPROCESS_STATE("0"); // 1 正常 0 失败
fields.setBUSINESS_STATE("1"); // 1 正常 0 失败
fields.setRECORD_TIME(new Date());
......
......@@ -3,7 +3,6 @@ package com.jianju.di.task;
import com.alibaba.fastjson.JSONObject;
import com.alibaba.fastjson.util.TypeUtils;
//import com.jianju.di.Manager.PostGISMannager;
import com.jianju.di.Manager.PostGISMannager;
import com.jianju.di.entity.Di;
import com.jianju.di.utils.FtpUtil;
......@@ -14,6 +13,7 @@ import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import java.io.File;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
......@@ -49,8 +49,10 @@ public class PullFileTask {
@Value("${ftp.olrremotepath}")
private String olrRemotePath;
@Value("${ftp.godasremotepath")
private String godasRemotePath;
@Value("${ftp.godas.pentadremotepath")
private String godasPentadRemotePath;
@Value("${ftp.godas.monthlyremotepath")
private String godasMonthlyRemotePath;
/**
* 数据库注入参数
......@@ -82,14 +84,73 @@ public class PullFileTask {
private static final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyyMMdd");
private static final SimpleDateFormat yearDateFormat = new SimpleDateFormat("yyyy");
private static final SimpleDateFormat yearMonthDateFormat = new SimpleDateFormat("MM");
private final FtpUtil.FtpConfig ftpConfig = new FtpUtil.FtpConfig().setHost(host).setPort(port).setUserName(userName).setPassword(password);
/**
*
* Monthly数据入库以及发送di
* 暂定为每月15晚八点运行获取上一个月的数据
*/
// @Scheduled(cron = "0 0 20 15 * ?") 目前ftp数据还未准备好,所以暂时先不进行运行
public void pullMonthlyGodasFileAndSendDi() throws Exception
{
Calendar c = Calendar.getInstance();
c.setTime(new Date());
//上一个月
c.add(Calendar.MONTH, -1);
String fileName = "godas.M." + yearMonthDateFormat.format(c.getTime())+".grb";
log.info("开始下载----" + fileName);
long beforeDown = System.currentTimeMillis();
boolean download = FtpUtil.download(ftpConfig, godasMonthlyRemotePath, fileName, localPath, fileName);
long afterDown;
if (download) {
log.info("下载成功" + fileName);
PostGISMannager postGIS = new PostGISMannager(dbHost, dbPort,
dbName, godasMonthlyTbName, dbUserName, dbPassword);
if (postGIS.insertGodasToPGMonth(godasMonthlyTbName, localPath + fileName) > 0) {
// 发送Di
afterDown = System.currentTimeMillis();
Date start = new Date(beforeDown);
long fileSize = new File(localPath,fileName).length();
String diJson = sendDi("godas-monthly数据入库","K.0436.0001.6666",c.getTime(), new Date(beforeDown), new Date(afterDown), fileName, "1",fileSize);
OkHttpClient httpClient = new OkHttpClient();
RequestBody requestBody = RequestBody.create(MediaType.parse("application/json; charset=utf-8"), diJson);
Request request = new Request.Builder()
.url(diUrl)
.post(requestBody)
.build();
//发送请求获取响应
try {
Response response = httpClient.newCall(request).execute();
//判断请求是否成功
if (response.isSuccessful()) {
//打印服务端返回结果
log.info("请求代理返回结果==========:" + response.body().string());
log.info("请求di的Json=========" + diJson);
}
} catch (IOException e) {
e.printStackTrace();
}
}
} else {
log.info("下载失败" + fileName);
}
}
/**
*pentad数据入库以及发送di
* 目前默认没玩8点获取前俩天的数据
* @throws Exception
*/
@Scheduled(cron = "0 0 20 * * ?")
// @Scheduled(cron = "0 0 20 * * ?") 目前ftp数据还未准备好,所以暂时先不进行运行
public void pullPentadGodasFileAndSendDi() throws Exception {
Calendar c = Calendar.getInstance();
c.setTime(new Date());
......@@ -102,7 +163,7 @@ public class PullFileTask {
fileName = "godas.P." + dateFormat.format(c.getTime()) + ".grb";
year = yearDateFormat.format(c.getTime());
beforeDown = System.currentTimeMillis();
download = FtpUtil.download(ftpConfig, godasRemotePath + year + "/", fileName, localPath, fileName);
download = FtpUtil.download(ftpConfig, godasPentadRemotePath + year + "/", fileName, localPath, fileName);
if (download) {
log.info("下载成功" + fileName);
......@@ -113,8 +174,8 @@ public class PullFileTask {
log.info("入库成功"+fileName);
afterDown = System.currentTimeMillis();
Date start = new Date(beforeDown);
String diJson = sendDi("godas数据入库","K.0436.0001.6666",c.getTime(), new Date(beforeDown), new Date(afterDown), fileName, "1");
long fileSize = new File(localPath,fileName).length();
String diJson = sendDi("godas-pentad数据入库","K.0436.0001.6666",c.getTime(), new Date(beforeDown), new Date(afterDown), fileName, "1",fileSize);
OkHttpClient httpClient = new OkHttpClient();
......@@ -143,10 +204,12 @@ public class PullFileTask {
}
//每日20:10执行;
/**
* 每日20:10执行;
* olr数据入库以及发送di
* @throws Exception
*/
@Scheduled(cron = "0 10 20 * * ?")
//@Scheduled(fixedRate = 10000)
//@PostConstruct
public void pullOlrFileAndSendDi() throws Exception {
Calendar c = Calendar.getInstance();
c.setTime(new Date());
......@@ -168,9 +231,8 @@ public class PullFileTask {
if (postGIS.insertOlrToPG(olrTbName, localPath + fileName) > 0) {
// 发送Di
afterDown = System.currentTimeMillis();
Date start = new Date(beforeDown);
String diJson = sendDi("olr数据入库","K.0502.0001.0001",c.getTime(), new Date(beforeDown), new Date(afterDown), fileName, "1");
long fileSize = new File(localPath,fileName).length();
String diJson = sendDi("olr数据入库","K.0502.0001.0001",c.getTime(), new Date(beforeDown), new Date(afterDown), fileName, "1",fileSize);
OkHttpClient httpClient = new OkHttpClient();
......@@ -199,7 +261,7 @@ public class PullFileTask {
}
private String sendDi(String Name, String DATA_TYPE, Date DATA_TIME, Date PROCESS_START_TIME, Date PROCESS_END_TIME, String FILE_NAME, String PROCESS_STATE) {
private String sendDi(String Name, String DATA_TYPE, Date DATA_TIME, Date PROCESS_START_TIME, Date PROCESS_END_TIME, String FILE_NAME, String PROCESS_STATE,long FILE_SIZE) {
List a = new ArrayList();
Di di = new Di();
Di.Fields fields = di.new Fields();
......@@ -220,7 +282,7 @@ public class PullFileTask {
fields.setPROCESS_END_TIME(PROCESS_END_TIME); //入库完成时间 或者失败的时间
fields.setFILE_NAME_O(FILE_NAME);
fields.setFILE_NAME_N(FILE_NAME);
fields.setFILE_SIZE("124560"); //单位B
fields.setFILE_SIZE(FILE_SIZE); //单位B
fields.setPROCESS_STATE(PROCESS_STATE); // 1 正常 0 失败
fields.setBUSINESS_STATE(PROCESS_STATE); // 1 正常 0 失败
fields.setRECORD_TIME(new Date());
......@@ -232,7 +294,7 @@ public class PullFileTask {
/**
* 对定时任务之前缺失的数据入库,但不发送di
* 对olr定时任务之前缺失的数据入库,但不发送di
*
* @throws Exception
*/
......@@ -278,6 +340,8 @@ public class PullFileTask {
public static void main(String[] args) {
}
......
......@@ -8,8 +8,10 @@ ftp:
username: ncc
password: ncc123
olrremotepath: /NOAA/NCEP/noaa18_olr/
godasremotepath: /NOAA/NCEP/GODAS/pentad/
localpath: /home/jianju/di/tempdata
godas:
pentadremotepath: /NOAA/NCEP/GODAS/pentad/
monthlyremotepath: /NOAA/NCEP/GODAS/monthly/
postgis:
host: 10.40.16.38
port: 5432
......@@ -17,9 +19,9 @@ postgis:
password: jianju@123@pg
dbname: test
olrtbname: noaadata
godast:
pentadtbname: 待定
monthlytbname: 待定
godas:
pentadtbname: godas
monthlytbname: godas
di:
url: http://127.0.0.1:29999/transfer
logging:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment