Skip to content

D
di
Commit e1f9dbba authored by Xuxiaodong's avatar Xuxiaodong
Browse files
Options

添加月数据入库代码

parent 5fa436dc
Hide whitespace changes
Inline Side-by-side
Showing with 384 additions and 108 deletions
src/main/java/com/jianju/di/DiApplication.java
View file @ e1f9dbba
......@@ -9,6 +9,7 @@ import org.springframework.scheduling.annotation.EnableScheduling;
import java.io.*;
@SpringBootApplication
@EnableScheduling
public class DiApplication {
......
src/main/java/com/jianju/di/Manager/PostGISMannager.java
View file @ e1f9dbba
......@@ -6,8 +6,10 @@ import visad.browser.Convert;
import java.io.*;
import java.sql.*;
import java.text.ParseException;
import java.text.ParsePosition;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
public class PostGISMannager {
private Connection c ;
......@@ -22,6 +24,36 @@ public class PostGISMannager {
m_tablename=tablename;
}
public void createLandPrecip(String tableName) throws SQLException
{
String sql="create table if not exists public."+tableName+
"(byteData bytea,"+
"time date,"+
"minlon numeric,"+
"maxlon numeric,"+
"maxlat numeric,"+
"minlat numeric,"+
"type char(50),"+
"dataCode char(50),"+
"geom geometry);";
stmt.executeUpdate(sql);
}
public void createCMAP(String tableName) throws SQLException
{
String sql="create table if not exists public."+tableName+
"(byteData bytea,"+
"time date,"+
"minlon numeric,"+
"maxlon numeric,"+
"maxlat numeric,"+
"minlat numeric,"+
"type char(50),"+
"dataCode char(50),"+
"geom geometry);";
stmt.executeUpdate(sql);
}
public void createOisstv2Mon(String tableName) throws SQLException
{
String sql="create table if not exists public."+tableName+
......@@ -70,12 +102,125 @@ public class PostGISMannager {
"geom geometry);";
stmt.executeUpdate(sql);
}
// public int insertLandPrecipToPG(String tableName,String dataPath) throws ParseException, IOException {
// int status=0;
// String geom;
// String type="precip";
// String datacode="REAN_NOAA_PRECIP_ORIGINAL";
// float minLat,maxLat,minLon,maxLon;
//
// Precip readCMAP=new Precip();
// float[][][] priecipData=readCMAP.readPrecip(dataPath);
// minLon=readCMAP.getMinlon();
// maxLat=readCMAP.getMaxlat();
// maxLon=readCMAP.getMaxlon();
// minLat=readCMAP.getMinlat();
// ArrayList<Date> arrayListsqlDate=readCMAP.getArrayListsqlDate();
// ArrayList<byte[]> arrayListbyte=readCMAP.getArrayListbyte();
// geom="POLYGON(("+minLon+" "+maxLat+","+maxLon+" "+maxLat+","+maxLon+" "+minLat+","+minLon+" "+minLat+","+minLon+" "+maxLat+"))";
//
//
// for(int i=0;i<arrayListsqlDate.size();i++)
// {
// PreparedStatement ps = null;
// ResultSet st = null;
// try
// {
// Date sqlDate= arrayListsqlDate.get(i);
// System.out.println("success connect");
// String sql ="insert into "+tableName+"(bytedata,time,minlon,maxlon,maxlat,minlat,type,dataCode,geom)" +
// " values(?,?,?,?,?,?,?,?,"+"'"+geom+"'"+")";
// try {
// ps = c.prepareStatement(sql);
// } catch (SQLException e) {
// e.printStackTrace();
// }
// float[][] tempData=priecipData[i];
// byte[] byteData= arrayListbyte.get(i);
// ps.setBytes(1, byteData);
// ps.setDate(2, sqlDate);
// ps.setFloat(3,minLon);
// ps.setFloat(4,maxLon);
// ps.setFloat(5,maxLat);
// ps.setFloat(6,minLat);
// ps.setString(7,type);
// ps.setString(8,datacode);
// status=ps.executeUpdate();
//
// }
// catch (Exception ex)
// {
// ex.printStackTrace(System.out);
// }
// }
//
// return status;
// }
// public int insertCMAPToPG(String tableName,String dataPath) throws ParseException, IOException {
// int status=0;
// String geom;
// String type="precip";
// String datacode="REAN_NOAA_PRECIP_ORIGINAL";
// float minLat,maxLat,minLon,maxLon;
//
// CMAP readCMAP=new CMAP();
// float[][][] priecipData=readCMAP.readCMAPData(dataPath);
// minLon=readCMAP.getMinlon();
// maxLat=readCMAP.getMaxlat();
// maxLon=readCMAP.getMaxlon();
// minLat=readCMAP.getMinlat();
// ArrayList<Date> arrayListsqlDate=readCMAP.getArrayListsqlDate();
// ArrayList<byte[]> arrayListbyte=readCMAP.getArrayListbyte();
// geom="POLYGON(("+minLon+" "+maxLat+","+maxLon+" "+maxLat+","+maxLon+" "+minLat+","+minLon+" "+minLat+","+minLon+" "+maxLat+"))";
//
//
// for(int i=0;i<arrayListsqlDate.size();i++)
// {
// PreparedStatement ps = null;
// ResultSet st = null;
// try
// {
// Date sqlDate= arrayListsqlDate.get(i);
// System.out.println("success connect");
// String sql ="insert into "+tableName+"(bytedata,time,minlon,maxlon,maxlat,minlat,type,dataCode,geom)" +
// " values(?,?,?,?,?,?,?,?,"+"'"+geom+"'"+")";
// try {
// ps = c.prepareStatement(sql);
// } catch (SQLException e) {
// e.printStackTrace();
// }
// float[][] tempData=priecipData[i];
// byte[] byteData= arrayListbyte.get(i);
// ps.setBytes(1, byteData);
// ps.setDate(2, sqlDate);
// ps.setFloat(3,minLon);
// ps.setFloat(4,maxLon);
// ps.setFloat(5,maxLat);
// ps.setFloat(6,minLat);
// ps.setString(7,type);
// ps.setString(8,datacode);
// status=ps.executeUpdate();
//
// }
// catch (Exception ex)
// {
// ex.printStackTrace(System.out);
// }
// }
//
// return status;
// }
// public int insertOisstv2MonToPG(String tableName,String dataPath,boolean ice) throws ParseException, IOException {
// int status=0;
// String geom;
// String type="oisst";
// String datacode="REAN_NOAA_OISST_ORIGINAL";
// if(ice) {
// type="ice";
// datacode="REAN_NOAA_ICE_ORIGINAL";
// }
//
// String time;
// float minLat,maxLat,minLon,maxLon;
// ICEandSST readICESST=new ICEandSST();
......@@ -84,7 +229,6 @@ public class PostGISMannager {
// maxLat=readICESST.getMaxlat();
// maxLon=readICESST.getMaxlon();
// minLat=readICESST.getMinlat();
// minLat=-minLat;
// ArrayList<Integer> arrayListTimeInt =readICESST.getArrayListTimeInt();
// int Time= arrayListTimeInt.get(0);
// geom="POLYGON(("+minLon+" "+maxLat+","+maxLon+" "+maxLat+","+maxLon+" "+minLat+","+minLon+" "+minLat+","+minLon+" "+maxLat+"))";
......@@ -122,6 +266,123 @@ public class PostGISMannager {
// }
// return status;
// }
public int insertGodasToPGMonth(String tableName,String gdasName) throws IOException, SQLException, ParseException {
String name="";
int lines=0;
String geom;
String type="u";
String type1="v";
String type2="sst";
String type3="u_flx";
String datacode="REAN_GODAS_MUL_PEN";
String time;
float minLat,maxLat,minLon,maxLon;
String mongoID = null;
int status=0;
maxLat= (float) 64.499;
minLat=-74;
minLon=1;
maxLon=(float)360;
int hou=1;
name = gdasName;
String[] temp=name.split("\\\\");
String filename=temp[temp.length-1];
String[] strDate=filename.split("\\.");
int intYear= Convert.getInt(strDate[2]);
if (name.endsWith(".grb")) {
hou++;
GribManager data = new GribManager(name);
float[] depath = data.getDepth();
float[][] sst;
float[][] v;
float[][] u;
float[][] u_flx;
byte[] bytessst;
byte[] bytesv;
byte[] bytesu;
byte[] bytesu_flx;
int lat = data.getLat();
int lon = data.getLon();
System.out.println("插入第" + intYear + "条数据"+name);
//datacode=s[7];
geom = "POLYGON((" + minLon + " " + maxLat + "," + maxLon + " " + maxLat + "," + maxLon + " " + minLat + "," + minLon + " " + minLat + "," + minLon + " " + maxLat + "))";
for (int j = 0; j < depath.length; j++) {
sst = data.getSST(j);
v = data.getSST(j);
u = data.getSST(j);
u_flx = data.getU_flx();
bytessst = floatArray2DToByteArray(sst, lat, lon);
bytesv = floatArray2DToByteArray(v, lat, lon);
bytesu = floatArray2DToByteArray(u, lat, lon);
bytesu_flx = floatArray2DToByteArray(u_flx, lat, lon);
PreparedStatement ps = null;
ResultSet st = null;
try {
String sql = "insert into " + tableName + "(bytedata,depth,time,minlon,maxlon,maxlat,minlat,type,dataCode,geom)" +
" values(?,?,?,?,?,?,?,?,?," + "'" + geom + "'" + ")";
try {
ps = c.prepareStatement(sql);
} catch (SQLException e) {
e.printStackTrace();
}
ps.setBytes(1, bytessst);
ps.setInt(2, (int) depath[j]);
ps.setInt(3, intYear);
ps.setFloat(4, minLon);
ps.setFloat(5, maxLon);
ps.setFloat(6, maxLat);
ps.setFloat(7, minLat);
ps.setString(8, "sst");
ps.setString(9, datacode);
ps.executeUpdate();
ps.setBytes(1, bytesv);
ps.setInt(2, (int) depath[j]);
ps.setInt(3, intYear);
ps.setFloat(4, minLon);
ps.setFloat(5, maxLon);
ps.setFloat(6, maxLat);
ps.setFloat(7, minLat);
ps.setString(8, "v");
ps.setString(9, datacode);
ps.executeUpdate();
ps.setBytes(1, bytesu);
ps.setInt(2, (int) depath[j]);
ps.setInt(3, intYear);
ps.setFloat(4, minLon);
ps.setFloat(5, maxLon);
ps.setFloat(6, maxLat);
ps.setFloat(7, minLat);
ps.setString(8, "u");
ps.setString(9, datacode);
ps.executeUpdate();
if (j == 0) {
ps.setBytes(1, bytesu_flx);
ps.setInt(2, (int) depath[j]);
ps.setInt(3, intYear);
ps.setFloat(4, minLon);
ps.setFloat(5, maxLon);
ps.setFloat(6, maxLat);
ps.setFloat(7, minLat);
ps.setString(8, "u_flx");
ps.setString(9, datacode);
status=ps.executeUpdate();
}
} catch (Exception ex) {
ex.printStackTrace(System.out);
}
}
lines++;
}
return status;
}
public int insertGodasToPG(String tableName,String gdasName) throws IOException, SQLException, ParseException {
String name="";
......@@ -146,8 +407,19 @@ public class PostGISMannager {
String[] temp=name.split("\\\\");
String filename=temp[temp.length-1];
String[] strDate=filename.split("\\.");
int intYear= Convert.getInt(strDate[2]);
int intYear= Convert.getInt(strDate[2].substring(0,5));
//int monthAndH=Convert.getInt(strDate[2].substring(6,8));
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyyMMdd"); // 日期格式
java.util.Date date = dateFormat.parse(strDate[2]); // 指定日期
Calendar ca = Calendar.getInstance();//创建一个日期实例
ca.setTime(date);//实例化一个日期
int H=ca.get(Calendar.DAY_OF_YEAR)/5;//获取是第多少天
intYear=intYear*10+H;
if (name.endsWith(".grb")) {
hou++;
GribManager data = new GribManager(name);
......@@ -418,4 +690,5 @@ public class PostGISMannager {
}
return out.toByteArray();
}
}
src/main/java/com/jianju/di/task/PullFileTask.java
View file @ e1f9dbba
......@@ -28,6 +28,11 @@ import java.util.List;
@Slf4j
public class PullFileTask {
/**
*
*/
@Value("${di.url}")
private String diUrl;
/**
* Ftp注入参数
*/
......@@ -61,154 +66,149 @@ public class PullFileTask {
@Value("${postgis.dbname}")
private String dbName;
//olr数据入库表名称
@Value("${postgis.olrtbname}")
private String olrTbName;
@Value("${postgis.godastbname}")
private String godasTbName;
//godasPentad数据入库表名称
@Value("${postgis.godas.pentadtbname}")
private String godasPentadTbName;
//godasMonthly数据入库表名称
@Value("${postgis.godas.monthlytbname}")
private String godasMonthlyTbName;
private static final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyyMMdd");
private static final SimpleDateFormat yearDateFormat = new SimpleDateFormat("yyyy");
private final FtpUtil.FtpConfig ftpConfig = new FtpUtil.FtpConfig().setHost(host).setPort(port).setUserName(userName).setPassword(password);
//每隔五天执行一次；
@Scheduled(cron ="0 0 0 0/5 * ?" )
public void pullGodasFileAndSendDi() throws Exception {
private final FtpUtil.FtpConfig ftpConfig = new FtpUtil.FtpConfig().setHost(host).setPort(port).setUserName(userName).setPassword(password);
/**
*
* @throws Exception
*/
@Scheduled(cron = "0 0 20 * * ?")
public void pullPentadGodasFileAndSendDi() throws Exception {
Calendar c = Calendar.getInstance();
c.setTime(new Date());
// -2 为ftp上传文件延迟的时间
c.add(Calendar.DATE, -2);
String year ;
String fileName = null;
long beforeDown = 0;
long afterDown = 0;
boolean download = false;
for (int i = 0; i <= 5; i++) {
fileName = "godas.P."+ dateFormat.format(c.getTime())+".grb";
year = yearDateFormat.format(c.getTime());
beforeDown = System.currentTimeMillis();
download = FtpUtil.download(ftpConfig, godasRemotePath + year + "/", fileName, localPath, fileName);
if(download) {
log.info("下载成功"+fileName);
break;
}
c.add(Calendar.DATE, -1);
}
if(download)
{
String year;
String fileName ;
long beforeDown, afterDown;
boolean download ;
fileName = "godas.P." + dateFormat.format(c.getTime()) + ".grb";
year = yearDateFormat.format(c.getTime());
beforeDown = System.currentTimeMillis();
download = FtpUtil.download(ftpConfig, godasRemotePath + year + "/", fileName, localPath, fileName);
if (download) {
log.info("下载成功" + fileName);
PostGISMannager postGIS = new PostGISMannager(dbHost, dbPort,
dbName, godasTbName, dbUserName, dbPassword);
postGIS.createTableGDAS(godasTbName);
if(postGIS.insertGodasToPG(godasTbName, localPath + fileName)>0){
// 发送Di -----------暂时修改到了这里 2020年4月2日19点02分
dbName, godasPentadTbName, dbUserName, dbPassword);
postGIS.createTableGDAS(godasPentadTbName);
if (postGIS.insertGodasToPG(godasPentadTbName, localPath + fileName) > 0) {
log.info("入库成功"+fileName);
afterDown = System.currentTimeMillis();
Date start = new Date(beforeDown);
Date start = new Date(beforeDown);
String diJson = sendDi(c.getTime(), new Date(beforeDown), new Date(afterDown), fileName, "1");
String diJson = sendDi("godas数据入库","K.0436.0001.6666",c.getTime(), new Date(beforeDown), new Date(afterDown), fileName, "1");
OkHttpClient httpClient = new OkHttpClient();
RequestBody requestBody = RequestBody.create(MediaType.parse("application/json; charset=utf-8"), diJson);
Request request = new Request.Builder()
.url("http://127.0.0.1:29999/transfer")
.url(diUrl)
.post(requestBody)
.build();
//发送请求获取响应
try {
Response response=httpClient.newCall(request).execute();
Response response = httpClient.newCall(request).execute();
//判断请求是否成功
if(response.isSuccessful()){
if (response.isSuccessful()) {
//打印服务端返回结果
log.info("请求代理返回结果==========："+response.body().string());
log.info("请求di的Json========="+diJson);
log.info("请求代理返回结果==========：" + response.body().string());
log.info("请求di的Json=========" + diJson);
}
} catch (IOException e) {
e.printStackTrace();
}
}
} else {
log.info("下载失败" + fileName);
}
else
{
log.info("下载失败"+fileName);
}
}
//每日20：10执行；
@Scheduled(cron ="0 10 20 * * ?" )
@Scheduled(cron = "0 10 20 * * ?")
//@Scheduled(fixedRate = 10000)
//@PostConstruct
public void pullOlrFileAndSendDi() throws Exception {
Calendar c = Calendar.getInstance();
c.setTime(new Date());
c.add(Calendar.DATE, -2);
String fileName = "#"+dateFormat.format(c.getTime());
String fileName = "#" + dateFormat.format(c.getTime());
log.info("开始下载----"+fileName);
log.info("开始下载----" + fileName);
long beforeDown = System.currentTimeMillis();
boolean download = FtpUtil.download(ftpConfig, olrRemotePath, fileName, localPath, fileName);
long afterDown;
if(download)
{
if (download) {
log.info("下载成功"+fileName);
PostGISMannager postGIS = new PostGISMannager(dbHost, dbPort,
dbName, olrTbName, dbUserName, dbPassword);
postGIS.createOlrDay(olrTbName);
log.info("下载成功" + fileName);
PostGISMannager postGIS = new PostGISMannager(dbHost, dbPort,
dbName, olrTbName, dbUserName, dbPassword);
postGIS.createOlrDay(olrTbName);
if (postGIS.insertOlrToPG(olrTbName, localPath + fileName) > 0) {
// 发送Di
afterDown = System.currentTimeMillis();
Date start = new Date(beforeDown);
String diJson = sendDi("olr数据入库","K.0502.0001.0001",c.getTime(), new Date(beforeDown), new Date(afterDown), fileName, "1");
OkHttpClient httpClient = new OkHttpClient();
RequestBody requestBody = RequestBody.create(MediaType.parse("application/json; charset=utf-8"), diJson);
Request request = new Request.Builder()
.url(diUrl)
.post(requestBody)
.build();
//发送请求获取响应
try {
Response response = httpClient.newCall(request).execute();
//判断请求是否成功
if (response.isSuccessful()) {
//打印服务端返回结果
log.info("请求代理返回结果==========：" + response.body().string());
log.info("请求di的Json=========" + diJson);
if(postGIS.insertOlrToPG(olrTbName, localPath + fileName)>0){
// 发送Di
afterDown = System.currentTimeMillis();
Date start = new Date(beforeDown);
String diJson = sendDi(c.getTime(), new Date(beforeDown), new Date(afterDown), fileName, "1");
OkHttpClient httpClient = new OkHttpClient();
RequestBody requestBody = RequestBody.create(MediaType.parse("application/json; charset=utf-8"), diJson);
Request request = new Request.Builder()
.url("http://127.0.0.1:29999/transfer")
.post(requestBody)
.build();
//发送请求获取响应
try {
Response response=httpClient.newCall(request).execute();
//判断请求是否成功
if(response.isSuccessful()){
//打印服务端返回结果
log.info("请求代理返回结果==========："+response.body().string());
log.info("请求di的Json========="+diJson);
}
} catch (IOException e) {
e.printStackTrace();
}
} catch (IOException e) {
e.printStackTrace();
}
}
else
{
log.info("下载失败"+fileName);
}
} else {
log.info("下载失败" + fileName);
}
}
private String sendDi(Date DATA_TIME,Date PROCESS_START_TIME, Date PROCESS_END_TIME , String FILE_NAME ,String PROCESS_STATE)
{
private String sendDi(String Name, String DATA_TYPE, Date DATA_TIME, Date PROCESS_START_TIME, Date PROCESS_END_TIME, String FILE_NAME, String PROCESS_STATE) {
List a = new ArrayList();
Di di = new Di();
Di.Fields fields = di.new Fields();
di.setName("OLR数据入库");
di.setName(Name);
di.setOccur_time(System.currentTimeMillis());
di.setMessage("文件级");
di.setType("RT.DPC.STATION.DI");
fields.setDATA_TYPE("K.0436.0001.9999");
fields.setDATA_TYPE_1("K.0436.0001.9999");
fields.setDATA_TYPE(DATA_TYPE);
fields.setDATA_TYPE_1(DATA_TYPE);
fields.setRECEIVE("CTS");
fields.setSEND("STDB");
fields.setTRAN_TIME(new Date());
......@@ -226,50 +226,50 @@ public class PullFileTask {
fields.setRECORD_TIME(new Date());
di.setFields(fields);
a.add(di);
TypeUtils.compatibleWithJavaBean =true;
return JSONObject.toJSONString(a);
TypeUtils.compatibleWithJavaBean = true;
return JSONObject.toJSONString(a);
}
/**
* 对定时任务之前缺失的数据入库，但不发送di
*
* @throws Exception
*/
// @PostConstruct
public void a() throws Exception {
// @PostConstruct
public void a() throws Exception {
FtpUtil.FtpConfig ftpConfig = new FtpUtil.FtpConfig().setHost(host).setPort(port).setUserName(userName).setPassword(password);
Date dateStart = dateFormat.parse("20200325");
Date dateEnd = dateFormat.parse("20200325");
Calendar c = Calendar.getInstance();
PostGISMannager postGIS = new PostGISMannager(dbHost, dbPort,
dbName, olrTbName, dbUserName, dbPassword);
while (dateStart.compareTo(dateEnd)<=0) {
while (dateStart.compareTo(dateEnd) <= 0) {
String fileName = "#" + dateFormat.format(dateStart.getTime());
log.info("开始下载----" + fileName);
boolean download = FtpUtil.download(ftpConfig, olrRemotePath, fileName, localPath, fileName);
if (download) {
log.info("下载成功-----"+ fileName);
log.info("下载成功-----" + fileName);
postGIS.createOlrDay(olrTbName);
if (postGIS.insertOlrToPG(olrTbName, localPath + fileName) > 0) {
log.info(fileName + "=======入库成功");
c.setTime(dateStart);
c.add(c.DATE, 1);//把dateBegin加上1天然后重新赋值给date1
dateStart = c.getTime();
} else {
log.info(fileName + "========入库失败");
}
else
{
log.info(fileName+"========入库失败");
}
}{
log.info(fileName+"========下载失败");
}
{
log.info(fileName + "========下载失败");
break;
}
}
}
// @PostConstruct
public void testDownload()
{
// @PostConstruct
public void testDownload() {
FtpUtil.FtpConfig ftpConfig = new FtpUtil.FtpConfig().setHost(host).setPort(port).setUserName(userName).setPassword(password);
boolean download = FtpUtil.download(ftpConfig, olrRemotePath, "#20200324", localPath, "#20200324");
......@@ -281,6 +281,4 @@ public class PullFileTask {
}
}
src/main/resources/application.yml
View file @ e1f9dbba
......@@ -17,7 +17,11 @@ postgis:
password: jianju@123@pg
dbname: test
olrtbname: noaadata
godastbname: 待定
godast:
pentadtbname: 待定
monthlytbname: 待定
di:
url: http://127.0.0.1:29999/transfer
logging:
file:
path: /home/jianju/di/logs
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment