innov
/
aggregation-platform
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

冷热区数据分为冷区一张表热区一张表存储

Browse Source
web_backend_develope
chenlw 10 years ago
parent 86436a12b8
commit 9dd1b0f019
14 changed files with 521 additions and 72 deletions
Show Stats Download Patch File Download Diff File

357
src/com/dao/mapper/hotDataDetailsMapper.xml
View File

File diff suppressed because it is too large Load Diff

11
src/com/dao/mapper/standard_data_details-mapper.xml
Unescape View File

@ -203,6 +203,17 @@ PUBLIC "-//ibatis.apache.org//DTD Mapper 3.0//EN"
ORDER BY a.area_code,a.sys_code,d.data_version
</select>
<select id="findHotDataByCity" resultType="com.platform.entities.CheckoutEntity" parameterType="com.platform.entities.PreDataInfo">
SELECT
<include refid="find_city_Column_List" />
FROM standard_data_details a left join collection_data_description l on a.area_code=l.regionalism_code AND a.sys_code=l.sys_code
left join hot_data_details d on a.area_code=d.regionalism_code AND a.sys_code=d.system_code
<where>
a.is_collection='是' AND a.city_name=#{cityName}
</where>
ORDER BY a.area_code,a.sys_code,d.data_version
</select>
<!-- 批量插入 -->
<select id="insertBatch" parameterType="java.util.List">
INSERT INTO default_data_description ( <include refid="Batch_Column_List" /> )

39
src/com/platform/dao/IHotDataInfoDao.java
Unescape View File

@ -0,0 +1,39 @@
package com.platform.dao;
import java.util.List;
import org.springframework.stereotype.Repository;
import com.platform.entities.DataInfoEntity;
import com.platform.form.PagerOptions;
@Repository(value = "hotDataInfoDao")
public interface IHotDataInfoDao {
int getLimitedDataCount(PagerOptions pagerOptions);
int getLimitedBeginId(PagerOptions pagerOptions);
List<DataInfoEntity> getLimitedDataInfoEntities(PagerOptions pagerOptions);
List<DataInfoEntity> getLimitedDataInfoByPage(PagerOptions pagerOptions);
List<String> getIdIsExist(List<Integer> list)throws Exception;
List<DataInfoEntity> findAll()throws Exception;
DataInfoEntity findById(int id)throws Exception;
List<DataInfoEntity> findByParam(DataInfoEntity data)throws Exception;
int countByDataPath(DataInfoEntity data)throws Exception;
int removes(List<Integer> list)throws Exception;
int save(DataInfoEntity data) throws Exception;
int update(DataInfoEntity data) throws Exception;
int updateExtract(DataInfoEntity data) throws Exception;
}

2
src/com/platform/dao/PreDataInfoDao.java
Unescape View File

@ -25,6 +25,8 @@ public interface PreDataInfoDao {
List<CheckoutEntity> findByCity(PreDataInfo pre)throws Exception;
List<CheckoutEntity> findHotDataByCity(PreDataInfo pre)throws Exception;
void insertBatch(List<PreDataInfo> list) throws Exception;
int update(PreDataInfo data) throws Exception;

3
src/com/platform/oracle/OracleConnector.java
Unescape View File

@ -68,8 +68,9 @@ public class OracleConnector {
.createStatement(ResultSet.TYPE_SCROLL_SENSITIVE,
ResultSet.CONCUR_UPDATABLE);
resultSet = statement.executeQuery(sql);
FileOperateHelper
.fileWrite(filePath+".log", sql+ "\r\n"+"OK \r\n");
.fileWrite(filePath+".log", sql+ "\r\n"+ "OK \r\n");
} catch (SQLException e) {
FileOperateHelper
.fileWrite(filePath+".log", sql+ "\r\n"+e.getMessage()+"\r\n");

151
src/com/platform/service/impl/CheckoutServiceImpl.java
Unescape View File

@ -119,79 +119,104 @@ public class CheckoutServiceImpl implements ICheckoutService {
}
@Override
public List<CheckoutEntity> findByCity(String city) throws Exception {
CheckoutEntity cksql = new CheckoutEntity();
Calendar c2 = Calendar.getInstance();
// 时间设置为 半年前的时间
c2.set(Calendar.MONTH, getMonBeforeHalfYear(c2.get(Calendar.MONTH)));
String time = DateForm.date2StringByDay(c2.getTime());
cksql.setCollectingTime(time);
cksql.setCityName(city);
Map<String,CheckoutEntity> resul = new HashMap<String,CheckoutEntity>();
Map<String,CheckoutEntity> nodata = new HashMap<String,CheckoutEntity>();
List<CheckoutEntity> list = preDataInfoDao.findByCity(cksql);
int length = list.size();
// 对所有数据分析
for (int i = 0; i < length; i++) {
CheckoutEntity preDataInfo = list.get(i);
preDataInfo.setId(i);
preDataInfo.setCollUpdate(isY(preDataInfo.getCollUpdate()));
if (preDataInfo.getDataVersion() < 1) {
preDataInfo.setCollection("否");
public List<CheckoutEntity> findByCity(String city) throws Exception {
CheckoutEntity cksql = new CheckoutEntity();
Calendar c2 = Calendar.getInstance();
// 时间设置为 半年前的时间
c2.set(Calendar.MONTH, getMonBeforeHalfYear(c2.get(Calendar.MONTH)));
String time = DateForm.date2StringByDay(c2.getTime());
cksql.setCollectingTime(time);
cksql.setCityName(city);
Map<String,CheckoutEntity> resul = new HashMap<String,CheckoutEntity>();
Map<String,CheckoutEntity> nodata = new HashMap<String,CheckoutEntity>();
//冷区数据
List<CheckoutEntity> list = preDataInfoDao.findByCity(cksql);
Map<String, List<CheckoutEntity>> coolDataMap = new HashMap<String, List<CheckoutEntity>>();
for (CheckoutEntity checkoutEntity : list) {
if(null == checkoutEntity.getCollectingTime()
|| c2.getTime().after(DateForm.string2DateByDay(checkoutEntity.getCollectingTime()))){
continue;
}
List<CheckoutEntity> tmpList = coolDataMap.get(checkoutEntity.getAreaCode()+"_"+checkoutEntity.getSysCode());
if (null == tmpList) {
tmpList = new ArrayList<CheckoutEntity>();
coolDataMap.put(checkoutEntity.getAreaCode()+"_"+checkoutEntity.getSysCode(),tmpList);
}
// 带数据的 系统
tmpList.add(checkoutEntity);
}
//热区数据
List<CheckoutEntity> hotDatalist = preDataInfoDao.findHotDataByCity(cksql);
for (CheckoutEntity checkoutEntity : hotDatalist) {
if (null != checkoutEntity.getCollectingTime()) {
//一段时间内的数据（半年）
if (c2.getTime().before(DateForm.string2DateByDay(checkoutEntity.getCollectingTime()))) {
// 带数据的 系统
resul.put(checkoutEntity.getAreaCode()+"_"+checkoutEntity.getSysCode()+"_"+checkoutEntity.getDataVersion(), checkoutEntity);
}
if (null != preDataInfo.getCollectingTime()) {
if (c2.getTime().before(DateForm.string2DateByDay(preDataInfo.getCollectingTime()))) {
resul.put(preDataInfo.getAreaCode()+"_"+preDataInfo.getSysCode()+"_"+preDataInfo.getDataId(), preDataInfo);
}
else {
preDataInfo.setDataId(0);
preDataInfo.setPath(null);
preDataInfo.setDataVersion(0);
preDataInfo.setExecResult(null);
preDataInfo.setExecResultLast(null);
preDataInfo.setPayResult(null);
preDataInfo.setPayResultLast(null);
preDataInfo.setCheckoutFlag(null);
preDataInfo.setCollection("否");
//data_id 已经为 0 （数据不存在时）
nodata.put(preDataInfo.getAreaCode()+"_"+preDataInfo.getSysCode(), preDataInfo);
}
// 半年之前的数据
else {
checkoutEntity.setDataId(0);
checkoutEntity.setPath(null);
checkoutEntity.setDataVersion(0);
checkoutEntity.setExecResult(null);
checkoutEntity.setExecResultLast(null);
checkoutEntity.setPayResult(null);
checkoutEntity.setPayResultLast(null);
checkoutEntity.setCheckoutFlag(null);
checkoutEntity.setCollection("否");
nodata.put(checkoutEntity.getAreaCode()+"_"+checkoutEntity.getSysCode()+"_"+checkoutEntity.getDataVersion(), checkoutEntity);
}
else{
nodata.put(preDataInfo.getAreaCode()+"_"+preDataInfo.getSysCode(), preDataInfo);
}
else {
// DataVersion = 0
checkoutEntity.setCollection("否");
nodata.put(checkoutEntity.getAreaCode()+"_"+checkoutEntity.getSysCode()+"_"+checkoutEntity.getDataVersion(), checkoutEntity);
}
}
// 将 nodata中的 无热区数据的 系统，在冷区中查询，如果有的话，将冷区数据放入 resul中， 而没有数据的 系统 的 key放入 keyList
List<String> noDataKeyList = new ArrayList<String>();
for ( String noDataKey : nodata.keySet()) {
CheckoutEntity checkData = nodata.get(noDataKey);
String coolDataMapKey = checkData.getAreaCode()+"_"+checkData.getSysCode();
if (coolDataMap.containsKey(coolDataMapKey)) {
List<CheckoutEntity> coolDatas = coolDataMap.get(coolDataMapKey);
for (CheckoutEntity checkoutEntity : coolDatas) {
resul.put(checkoutEntity.getAreaCode()+"_"+checkoutEntity.getSysCode()+"_"+ checkoutEntity.getDataVersion(), checkoutEntity);
}
}
List<String> keyList = new ArrayList<String>();
//记录 不存在数据 在 nodata的 key值
for (String nodatakey : nodata.keySet()) {
keyList.add(nodatakey);
else {
//记录 不存在数据 在 nodata的 key值
noDataKeyList.add(noDataKey);
}
List<CheckoutEntity> resultList = new ArrayList<CheckoutEntity>();
for (String key : resul.keySet()) {
//如果有 校验失败的结果则 为否
CheckoutEntity tmp = resul.get(key);
if(!Constant.CHECKOUT_STATUS_ZERO.equals(tmp.getPayResultLast()) && !Constant.CHECKOUT_STATUS_ZERO.equals(tmp.getExecResultLast())){
if (Constant.CHECKOUT_STATUS_FOUR.equals(tmp.getPayResultLast()) || Constant.CHECKOUT_STATUS_FOUR.equals(tmp.getExecResultLast())) {
tmp.setCheckResult(Constant.ORACLE_CHECK_REULT_ONE);
}
else if(!Constant.CHECKOUT_STATUS_TWO.equals(tmp.getPayResultLast()) && !Constant.CHECKOUT_STATUS_TWO.equals(tmp.getExecResultLast())){
tmp.setCheckResult(Constant.ORACLE_CHECK_REULT_ZERO);
}
}
List<CheckoutEntity> resultList = new ArrayList<CheckoutEntity>();
//带数据的
for (String key : resul.keySet()) {
//如果有 校验失败的结果则 为否
CheckoutEntity tmp = resul.get(key);
if(!Constant.CHECKOUT_STATUS_ZERO.equals(tmp.getPayResultLast()) && !Constant.CHECKOUT_STATUS_ZERO.equals(tmp.getExecResultLast())){
if (Constant.CHECKOUT_STATUS_FOUR.equals(tmp.getPayResultLast()) || Constant.CHECKOUT_STATUS_FOUR.equals(tmp.getExecResultLast())) {
tmp.setCheckResult(Constant.ORACLE_CHECK_REULT_ONE);
}
resultList.add(resul.get(key));
for (String nodatakey : keyList) {
if (key.contains(nodatakey)) {
nodata.remove(nodatakey);
}
else if(!Constant.CHECKOUT_STATUS_TWO.equals(tmp.getPayResultLast()) && !Constant.CHECKOUT_STATUS_TWO.equals(tmp.getExecResultLast())){
tmp.setCheckResult(Constant.ORACLE_CHECK_REULT_ZERO);
}
}
for (String key : nodata.keySet()) {
resultList.add(nodata.get(key));
resultList.add(resul.get(key));
}
//无数据的 系统
for (String nodatakey : noDataKeyList) {
if (nodata.keySet().contains(nodatakey)) {
CheckoutEntity checkoutEntity = nodata.get(nodatakey);
checkoutEntity.setCollection("否");
resultList.add(checkoutEntity);
}
return resultList;
}
return resultList;
}
@Override
public Map<String, PreDataInfo> findDetail(CheckoutEntity ck)

11
src/com/platform/service/impl/DataInfoServiceImp.java
Unescape View File

@ -15,6 +15,7 @@ import org.springframework.ui.ModelMap;
import com.base.Custom4exception;
import com.base.CustomException;
import com.platform.dao.DataInfoDao;
import com.platform.dao.IHotDataInfoDao;
import com.platform.entities.DataInfoEntity;
import com.platform.form.PagerOptions;
import com.platform.http.gfs.RemoveData;
@ -30,6 +31,9 @@ public class DataInfoServiceImp implements DataInfoService {
@Resource(name = "dataInfoDao")
private DataInfoDao dfdDao;
@Resource(name = "hotDataInfoDao")
private IHotDataInfoDao hotDataInfoDao;
private RemoveData removedata = new RemoveData();
public void setDfdDao(DataInfoDao dfdDao) {
@ -92,7 +96,12 @@ public class DataInfoServiceImp implements DataInfoService {
}
List<DataInfoEntity> result = null;
try{
result = dfdDao.getLimitedDataInfoByPage(pagerOptions);
if ("1".equals(pagerOptions.getMark())) {
result = dfdDao.getLimitedDataInfoByPage(pagerOptions);
}
else {
result = hotDataInfoDao.getLimitedDataInfoByPage(pagerOptions);
}
}catch(Exception e){
log.error(e.getMessage());
}

2
src/com/platform/service/thread/ThreadCheckoutStandardOracle.java
Unescape View File

@ -39,7 +39,7 @@ public class ThreadCheckoutStandardOracle extends Thread {
private DataInfoDao dataInfoDao;
public ThreadCheckoutStandardOracle(DataInfoDao dataInfoDao) {
this.setDaemon(true);
this.client = new SimpleKubeClient();
this.dataInfoDao = dataInfoDao;
}

1
src/com/platform/service/thread/ThreadExtractOracle.java
Unescape View File

@ -20,6 +20,7 @@ public class ThreadExtractOracle extends Thread {
* @param OracleExtract
*/
public ThreadExtractOracle(oracleForm form, IOracleExtractService OracleExtract) {
this.setDaemon(true);
this.OracleExtract = OracleExtract;
this.form = form;
}

1
src/com/platform/service/thread/ThreadExtractStandardOracle.java
Unescape View File

@ -30,6 +30,7 @@ public class ThreadExtractStandardOracle extends Thread {
* @param OracleExtract
*/
public ThreadExtractStandardOracle(oracleForm form, IOracleExtractService OracleExtract) {
this.setDaemon(true);
this.OracleExtract = OracleExtract;
this.form = form;
}

1
src/com/platform/service/thread/ThreadExtractStandardSqlServer.java
Unescape View File

@ -43,6 +43,7 @@ public class ThreadExtractStandardSqlServer extends Thread{
private DataInfoDao dataInfoDao;
public ThreadExtractStandardSqlServer(GatherOracleInfo oracleConnect, List<CheckoutEntity> Extractlist, DataInfoDao dataInfoDao) {
this.setDaemon(true);
this.oracleConnect = oracleConnect;
this.Extractlist = Extractlist;
this.dataInfoDao = dataInfoDao;

11
src/com/platform/service/thread/ThreadMoveData.java
Unescape View File

@ -17,6 +17,7 @@ import org.springframework.stereotype.Component;
import com.base.TaskOperateData;
import com.platform.dao.DataInfoDao;
import com.platform.dao.DataInfoMoveTmpDao;
import com.platform.dao.IHotDataInfoDao;
import com.platform.entities.DataInfoEntity;
import com.platform.entities.DataInfoEntityMoveTmp;
import com.platform.glusterfs.CheckoutMD5;
@ -36,8 +37,8 @@ public class ThreadMoveData{
public final static Logger log = Logger.getLogger(ThreadMoveData.class);
@Resource(name = "dataInfoDao")
private DataInfoDao dataInfoDao;
@Resource(name = "hotDataInfoDao")
private IHotDataInfoDao hotDataInfoDao;
/**
* 迁移
@ -336,7 +337,7 @@ public class ThreadMoveData{
data.setPayResult(dataMove.getPayResult());
data.setExecResult(dataMove.getExecResult());
data.setId(0);
dataInfoDao.save(data);
hotDataInfoDao.save(data);
}
else {
dataMove.setCompleteStatus("2");
@ -384,8 +385,8 @@ public class ThreadMoveData{
data.setExecResult(dataMove.getExecResult());
data.setId(0);
try {
if (dataInfoDao.countByDataPath(data) == 0) {
dataInfoDao.save(data);
if (hotDataInfoDao.countByDataPath(data) == 0) {
hotDataInfoDao.save(data);
}
} catch (Exception e) {
log.error(e);

1
src/com/platform/service/thread/TreadMoveData2Start.java
Unescape View File

@ -18,6 +18,7 @@ public class TreadMoveData2Start extends Thread{
private IMoveDataService dataInfoMove= new MoveDataServiceImpl();
public TreadMoveData2Start() {
this.setDaemon(true);
}
@Override

2
src/com/platform/utils/Constant.java
Unescape View File

@ -36,7 +36,7 @@ public class Constant {
+ "#这个必须要，否则会在文件名中有空格时出错\nINIT_PATH=\".\";\nergodic \\$1\n";
/**
* volume 获取的线程休眠时间
* 迁移数据时，并发迁移数量
*/
public final static int moveFileMaxNum = 1;

Write Preview
Loading…
Cancel
Save