Skip to content

D
di
Commit 9ea7d2ab authored by 续晓东's avatar 续晓东 🤺
Browse files
Options

完善业务流程,添加启动脚本

parent 5d9cb8a3
Hide whitespace changes
Inline Side-by-side
Showing with 160 additions and 76 deletions
README.md
View file @ 9ea7d2ab
# di # di
气象数据定时入库并且发送Di 气象数据定时入库并且发送Di
\ No newline at end of file ### 部署至气象10.40.16.38 端口号30000
\ No newline at end of file
pom.xml
View file @ 9ea7d2ab
...@@ -18,6 +18,32 @@ ...@@ -18,6 +18,32 @@
<java.version>1.8</java.version> <java.version>1.8</java.version>
</properties> </properties>
<repositories>
<repository>
<!-- id必须唯一 -->
<id>jboss-repository</id>
<!-- 见名知意即可 -->
<name>jboss repository</name>
<!-- 仓库的url地址 -->
<url>http://repository.jboss.org/nexus/content/groups/public-jboss/</url>
</repository>
<repository>
<!-- id必须唯一 -->
<id>aliyun-repository</id>
<!-- 见名知意即可 -->
<name>aliyun repository</name>
<!-- 仓库的url地址 -->
<url>http://maven.aliyun.com/nexus/content/groups/public/</url>
</repository>
<repository>
<!-- id必须唯一 -->
<id>boundlessgeo</id>
<!-- 见名知意即可 -->
<name>boundlessgeo</name>
<!-- 仓库的url地址 -->
<url>https://repo.boundlessgeo.com/main/</url>
</repository>
</repositories>
<dependencies> <dependencies>
<dependency> <dependency>
<groupId>org.springframework.boot</groupId> <groupId>org.springframework.boot</groupId>
......
src/main/java/com/jianju/di/DiApplication.java
View file @ 9ea7d2ab
...@@ -2,6 +2,7 @@ package com.jianju.di; ...@@ -2,6 +2,7 @@ package com.jianju.di;
import org.springframework.boot.SpringApplication; import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.scheduling.annotation.EnableScheduling; import org.springframework.scheduling.annotation.EnableScheduling;
...@@ -12,8 +13,10 @@ import java.io.*; ...@@ -12,8 +13,10 @@ import java.io.*;
@EnableScheduling @EnableScheduling
public class DiApplication { public class DiApplication {
public static void main(String[] args) throws IOException { public static void main(String[] args) throws IOException {
SpringApplication.run(DiApplication.class, args); SpringApplication.run(DiApplication.class, args);
} }
} }
src/main/java/com/jianju/di/Manager/PostGISMannager.java
View file @ 9ea7d2ab
package com.jianju.di.Manager; package com.jianju.di.Manager;
import DataBaseManager.GribManager;
import DataBaseManager.ICEandSST;
import DataBaseManager.MongodbManager;
import visad.browser.Convert; import visad.browser.Convert;
import java.io.*; import java.io.*;
...@@ -73,57 +71,57 @@ public class PostGISMannager { ...@@ -73,57 +71,57 @@ public class PostGISMannager {
stmt.executeUpdate(sql); stmt.executeUpdate(sql);
} }
public int insertOisstv2MonToPG(String tableName,String dataPath,boolean ice) throws ParseException, IOException { // public int insertOisstv2MonToPG(String tableName,String dataPath,boolean ice) throws ParseException, IOException {
int status=0; // int status=0;
String geom; // String geom;
String type="oisst"; // String type="oisst";
String datacode="REAN_NOAA_OISST_ORIGINAL"; // String datacode="REAN_NOAA_OISST_ORIGINAL";
String time; // String time;
float minLat,maxLat,minLon,maxLon; // float minLat,maxLat,minLon,maxLon;
ICEandSST readICESST=new ICEandSST(); // ICEandSST readICESST=new ICEandSST();
float[] iceData=readICESST.readIceTxtData(dataPath,ice); // float[] iceData=readICESST.readIceTxtData(dataPath,ice);
minLon=readICESST.getMinlon(); // minLon=readICESST.getMinlon();
maxLat=readICESST.getMaxlat(); // maxLat=readICESST.getMaxlat();
maxLon=readICESST.getMaxlon(); // maxLon=readICESST.getMaxlon();
minLat=readICESST.getMinlat(); // minLat=readICESST.getMinlat();
minLat=-minLat; // minLat=-minLat;
ArrayList<Integer> arrayListTimeInt =readICESST.getArrayListTimeInt(); // ArrayList<Integer> arrayListTimeInt =readICESST.getArrayListTimeInt();
int Time= arrayListTimeInt.get(0); // int Time= arrayListTimeInt.get(0);
geom="POLYGON(("+minLon+" "+maxLat+","+maxLon+" "+maxLat+","+maxLon+" "+minLat+","+minLon+" "+minLat+","+minLon+" "+maxLat+"))"; // geom="POLYGON(("+minLon+" "+maxLat+","+maxLon+" "+maxLat+","+maxLon+" "+minLat+","+minLon+" "+minLat+","+minLon+" "+maxLat+"))";
String[] arrName=dataPath.split("\\\\"); // String[] arrName=dataPath.split("\\\\");
String name=arrName[arrName.length-1]; // String name=arrName[arrName.length-1];
PreparedStatement ps = null; // PreparedStatement ps = null;
ResultSet st = null; // ResultSet st = null;
//
try // try
{ // {
System.out.println("success connect"); // System.out.println("success connect");
String sql ="insert into "+tableName+"(bytedata,name,time,minlon,maxlon,maxlat,minlat,type,dataCode,geom)" + // String sql ="insert into "+tableName+"(bytedata,name,time,minlon,maxlon,maxlat,minlat,type,dataCode,geom)" +
" values(?,?,?,?,?,?,?,?,?,"+"'"+geom+"'"+")"; // " values(?,?,?,?,?,?,?,?,?,"+"'"+geom+"'"+")";
try { // try {
ps = c.prepareStatement(sql); // ps = c.prepareStatement(sql);
} catch (SQLException e) { // } catch (SQLException e) {
e.printStackTrace(); // e.printStackTrace();
} // }
byte[] byteData=floatArrayToByteArray(iceData); // byte[] byteData=floatArrayToByteArray(iceData);
ps.setBytes(1, byteData); // ps.setBytes(1, byteData);
ps.setString(2,name); // ps.setString(2,name);
ps.setInt(3, Time); // ps.setInt(3, Time);
ps.setFloat(4,minLon); // ps.setFloat(4,minLon);
ps.setFloat(5,maxLon); // ps.setFloat(5,maxLon);
ps.setFloat(6,maxLat); // ps.setFloat(6,maxLat);
ps.setFloat(7,minLat); // ps.setFloat(7,minLat);
ps.setString(8,type); // ps.setString(8,type);
ps.setString(9,datacode); // ps.setString(9,datacode);
status=ps.executeUpdate(); // status=ps.executeUpdate();
//
} // }
catch (Exception ex) // catch (Exception ex)
{ // {
ex.printStackTrace(System.out); // ex.printStackTrace(System.out);
} // }
return status; // return status;
} // }
public int insertGodasToPG(String tableName,String gdasName) throws IOException, SQLException, ParseException { public int insertGodasToPG(String tableName,String gdasName) throws IOException, SQLException, ParseException {
String name=""; String name="";
......
src/main/java/com/jianju/di/Manager/Test.java
View file @ 9ea7d2ab
...@@ -14,16 +14,16 @@ public class Test { ...@@ -14,16 +14,16 @@ public class Test {
public static void main(String[] arg) throws ClassNotFoundException, SQLException, IOException, SQLException, IOException, InvalidRangeException, ParserConfigurationException, SAXException, TransformerException, ParseException { public static void main(String[] arg) throws ClassNotFoundException, SQLException, IOException, SQLException, IOException, InvalidRangeException, ParserConfigurationException, SAXException, TransformerException, ParseException {
//数据入库Test // 数据入库Test
// MongodbManager m_MyMongoDB = new MongodbManager("10.40.16.39", 27088, "Data", "day"); // MongodbManager m_MyMongoDB = new MongodbManager("10.40.16.39", 27088, "Data", "day");
// PostGISMannager postGIS = new PostGISMannager("localhost", "5432", // PostGISMannager postGIS = new PostGISMannager("localhost", "5432",
// "postgis_25_sample", "GODAS", "postgres", "123456"); // "postgis_25_sample", "GODAS", "postgres", "123456");
// postGIS.createTableGDAS("GODAS"); // postGIS.createTableGDAS("GODAS");
// postGIS.insertGodasToPG("GODAS", "E:\\NewWeather\\godas.P.20200105.grb"); // postGIS.insertGodasToPG("GODAS", "E:\\NewWeather\\godas.P.20200105.grb");
//
PostGISMannager postGIS = new PostGISMannager("localhost", "5432", // PostGISMannager postGIS = new PostGISMannager("localhost", "5432",
"postgis_25_sample", "GODAS", "postgres", "123456"); // "postgis_25_sample", "GODAS", "postgres", "123456");
postGIS.createOlrDay("Olr"); // postGIS.createOlrDay("Olr");
postGIS.insertOlrToPG("olr","E:\\NOAA\\#20050718"); // postGIS.insertOlrToPG("olr","E:\\NOAA\\#20050718");
} }
} }
src/main/java/com/jianju/di/task/PullFileTask.java
View file @ 9ea7d2ab
...@@ -4,9 +4,11 @@ package com.jianju.di.task; ...@@ -4,9 +4,11 @@ package com.jianju.di.task;
import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.JSONObject;
import com.alibaba.fastjson.util.TypeUtils; import com.alibaba.fastjson.util.TypeUtils;
import com.fasterxml.jackson.databind.util.JSONPObject; import com.fasterxml.jackson.databind.util.JSONPObject;
//import com.jianju.di.Manager.PostGISMannager;
import com.jianju.di.Manager.PostGISMannager; import com.jianju.di.Manager.PostGISMannager;
import com.jianju.di.entity.Di; import com.jianju.di.entity.Di;
import com.jianju.di.utils.FtpUtil; import com.jianju.di.utils.FtpUtil;
import lombok.Data;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import okhttp3.*; import okhttp3.*;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
...@@ -14,8 +16,11 @@ import org.springframework.scheduling.annotation.Scheduled; ...@@ -14,8 +16,11 @@ import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import javax.annotation.PostConstruct;
import java.io.IOException; import java.io.IOException;
import java.sql.SQLException;
import java.text.DateFormat; import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat; import java.text.SimpleDateFormat;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Calendar; import java.util.Calendar;
...@@ -65,28 +70,28 @@ public class PullFileTask { ...@@ -65,28 +70,28 @@ public class PullFileTask {
//每日0点与中午12点执行； //每日23点执行；
//@Scheduled(cron ="0 0 19 * * ?" ) @Scheduled(cron ="0 0 23 * * ?" )
@Scheduled(fixedRate = 10000) //@Scheduled(fixedRate = 10000)
public void PullFileAndSendDi() throws Exception { public void PullFileAndSendDi() throws Exception {
FtpUtil.FtpConfig ftpConfig = new FtpUtil.FtpConfig().setHost(host).setPort(port).setUserName(userName).setPassword(password); FtpUtil.FtpConfig ftpConfig = new FtpUtil.FtpConfig().setHost(host).setPort(port).setUserName(userName).setPassword(password);
Calendar c = Calendar.getInstance(); Calendar c = Calendar.getInstance();
c.setTime(new Date()); c.setTime(new Date());
c.add(Calendar.DATE, -2); c.add(Calendar.DATE, -1);
String fileName = "#"+dateFormat.format(c.getTime()); String fileName = "#"+dateFormat.format(c.getTime());
log.info("开始下载----"+fileName); log.info("开始下载----"+fileName);
long beforeDown = System.currentTimeMillis(); long beforeDown = System.currentTimeMillis();
boolean download = FtpUtil.download(ftpConfig, remotePath, fileName, localPath, fileName); boolean download = FtpUtil.download(ftpConfig, remotePath, fileName, localPath, fileName);
long afterDown = System.currentTimeMillis(); long afterDown;
if(download) if(download)
{ {
log.info("下载成功"); log.info("下载成功"+fileName);
PostGISMannager postGIS = new PostGISMannager(dbHost, dbPort, PostGISMannager postGIS = new PostGISMannager(dbHost, dbPort,
dbName, tbName, dbUserName, dbPassword); dbName, tbName, dbUserName, dbPassword);
postGIS.createOlrDay(tbName); postGIS.createOlrDay(tbName);
if(postGIS.insertOlrToPG(tbName, localPath + fileName)>0){ if(postGIS.insertOlrToPG(tbName, localPath + fileName)>0){
//发送Di // 发送Di
afterDown = System.currentTimeMillis(); afterDown = System.currentTimeMillis();
Date start = new Date(beforeDown); Date start = new Date(beforeDown);
...@@ -105,7 +110,7 @@ public class PullFileTask { ...@@ -105,7 +110,7 @@ public class PullFileTask {
//判断请求是否成功 //判断请求是否成功
if(response.isSuccessful()){ if(response.isSuccessful()){
//打印服务端返回结果 //打印服务端返回结果
log.info("请求代理返回结果+："+response.body().string()); log.info("请求代理返回结果==========："+response.body().string());
} }
} catch (IOException e) { } catch (IOException e) {
...@@ -115,7 +120,7 @@ public class PullFileTask { ...@@ -115,7 +120,7 @@ public class PullFileTask {
} }
else else
{ {
log.info("下载失败"); log.info("下载失败"+fileName);
} }
} }
...@@ -141,7 +146,7 @@ public class PullFileTask { ...@@ -141,7 +146,7 @@ public class PullFileTask {
fields.setPROCESS_END_TIME(PROCESS_END_TIME); //入库完成时间 或者失败的时间 fields.setPROCESS_END_TIME(PROCESS_END_TIME); //入库完成时间 或者失败的时间
fields.setFILE_NAME_O(FILE_NAME); fields.setFILE_NAME_O(FILE_NAME);
fields.setFILE_NAME_N(FILE_NAME); fields.setFILE_NAME_N(FILE_NAME);
fields.setFILE_SIZE("44032"); //单位B fields.setFILE_SIZE("124560"); //单位B
fields.setPROCESS_STATE(PROCESS_STATE); // 1 正常 0 失败 fields.setPROCESS_STATE(PROCESS_STATE); // 1 正常 0 失败
fields.setBUSINESS_STATE(PROCESS_STATE); // 1 正常 0 失败 fields.setBUSINESS_STATE(PROCESS_STATE); // 1 正常 0 失败
fields.setRECORD_TIME(new Date()); fields.setRECORD_TIME(new Date());
...@@ -152,5 +157,48 @@ public class PullFileTask { ...@@ -152,5 +157,48 @@ public class PullFileTask {
} }
// @PostConstruct
public void a() throws Exception {
FtpUtil.FtpConfig ftpConfig = new FtpUtil.FtpConfig().setHost(host).setPort(port).setUserName(userName).setPassword(password);
Date dateStart = dateFormat.parse("20191119");
Date dateEnd = dateFormat.parse("20191119");
Calendar c = Calendar.getInstance();
PostGISMannager postGIS = new PostGISMannager(dbHost, dbPort,
dbName, tbName, dbUserName, dbPassword);
while (dateStart.compareTo(dateEnd)<=0) {
String fileName = "#" + dateFormat.format(dateStart.getTime());
log.info("开始下载----" + fileName);
boolean download = FtpUtil.download(ftpConfig, remotePath, fileName, localPath, fileName);
if (download) {
log.info("下载成功-----"+ fileName);
postGIS.createOlrDay(tbName);
if (postGIS.insertOlrToPG(tbName, localPath + fileName) > 0) {
log.info(fileName + "=======入库成功");
c.setTime(dateStart);
c.add(c.DATE, 1);//把dateBegin加上1天然后重新赋值给date1
dateStart = c.getTime();
}
else
{
log.info(fileName+"========入库失败");
}
}{
log.info(fileName+"========下载失败");
break;
}
}
}
// @PostConstruct
public void b()
{
FtpUtil.FtpConfig ftpConfig = new FtpUtil.FtpConfig().setHost(host).setPort(port).setUserName(userName).setPassword(password);
boolean download = FtpUtil.download(ftpConfig, remotePath, "#20200324", localPath, "#20200324");
System.out.println(download+"asdasd");
}
} }
src/main/resources/application.yml
View file @ 9ea7d2ab
spring: spring:
application: application:
name: di name: di
...@@ -7,11 +8,16 @@ ftp: ...@@ -7,11 +8,16 @@ ftp:
username: ncc username: ncc
password: ncc123 password: ncc123
remotepath: /NOAA/NCEP/noaa18_olr/ remotepath: /NOAA/NCEP/noaa18_olr/
localpath: C:\Users\D\Desktop\证书\ localpath: /home/jianju/di/tempdata
postgis: postgis:
host: 10.40.16.38 host: 10.40.16.38
port: 5432 port: 5432
username: username: postgres
password: password: jianju@123@pg
dbname: Data dbname: test
tbname: day tbname: noaadata
\ No newline at end of file logging:
file:
path: /home/jianju/di/logs
server:
port: 30000
startup.sh 0 → 100644
View file @ 9ea7d2ab
#!/bin/sh
java -server -Xms4096m -Xmx4096m -Xmn1920m -jar ./di-0.0.1-SNAPSHOT.jar >/dev/null 2>&1 &
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment