Commit 80c9d172cfd7caa8a7081169b347ee2aa1974ce5
1 parent
65efc647
浦东调度车辆信息同步etl更新
车辆库命名参数url由程序传入，修改配置文件和相关代码 1、修改datatools下的config-*.properties所有配置文件，添加car_interface参数 2、修改DataToolsProperties.java设定car_interface属性 3、修改VehicleDataSyncTaskThread.java设置ktr命名参数url 4、修改VehicleDataSyncTaskThread.java，内部判断车辆自编号参数，时间范围参数的有效性和优先级，具体程序中和ktr中都有说明
Showing
8 changed files
with
101 additions
and
35 deletions
src/main/java/com/bsth/service/schedule/datasync/VehicleDataSyncTaskServiceImpl.java
| ... | ... | @@ -230,7 +230,7 @@ public class VehicleDataSyncTaskServiceImpl extends BServiceImpl<VehicleDataSync |
| 230 | 230 | try { |
| 231 | 231 | Path rootPath = VehicleDataSyncTaskThread.getLogbackFilePath(); |
| 232 | 232 | Path logFilePath = Paths.get(rootPath.toString(), "datasync", "vehicle", fileName); |
| 233 | - return logFilePath.toFile(); | |
| 233 | + file = logFilePath.toFile(); | |
| 234 | 234 | } catch (Exception exp) { |
| 235 | 235 | exp.printStackTrace(); |
| 236 | 236 | } | ... | ... |
src/main/java/com/bsth/service/schedule/datasync/task/VehicleDataSyncTaskThread.java
| ... | ... | @@ -3,6 +3,7 @@ package com.bsth.service.schedule.datasync.task; |
| 3 | 3 | import ch.qos.logback.classic.LoggerContext; |
| 4 | 4 | import ch.qos.logback.core.FileAppender; |
| 5 | 5 | import com.bsth.entity.schedule.datasync.VehicleDataSyncTask; |
| 6 | +import com.bsth.service.schedule.exception.ScheduleException; | |
| 6 | 7 | import com.bsth.service.schedule.utils.DataToolsProperties; |
| 7 | 8 | import com.bsth.service.schedule.utils.DataToolsServiceImpl; |
| 8 | 9 | import lombok.AllArgsConstructor; |
| ... | ... | @@ -27,6 +28,7 @@ import java.io.PrintWriter; |
| 27 | 28 | import java.io.StringWriter; |
| 28 | 29 | import java.nio.file.Path; |
| 29 | 30 | import java.nio.file.Paths; |
| 31 | +import java.util.Date; | |
| 30 | 32 | import java.util.HashMap; |
| 31 | 33 | import java.util.Map; |
| 32 | 34 | import java.util.concurrent.CountDownLatch; |
| ... | ... | @@ -74,11 +76,24 @@ public class VehicleDataSyncTaskThread implements Runnable { |
| 74 | 76 | dataToolsProperties.getVehicleDatasyncktr()).toURI()); |
| 75 | 77 | // 2、创建命名参数 |
| 76 | 78 | Map<String, String> ktrNamedParams = new HashMap<>(); |
| 77 | - ktrNamedParams.put("paramClzbh", vehicleDataSyncTask.getParamClzbh()); | |
| 78 | - ktrNamedParams.put("paramFrom", vehicleDataSyncTask.getParamFrom() == null ? | |
| 79 | - null : DateFormatUtils.format(vehicleDataSyncTask.getParamFrom(), "yyyy-MM-dd") + " 00:00:00"); | |
| 80 | - ktrNamedParams.put("paramTo", vehicleDataSyncTask.getParamTo() == null ? | |
| 81 | - null : DateFormatUtils.format(vehicleDataSyncTask.getParamTo(), "yyyy-MM-dd") + " 23:59:59"); | |
| 79 | + // 2-1、车辆库车辆信息url参数 | |
| 80 | + ktrNamedParams.put("url", this.dataToolsProperties.getCarsInterface()); | |
| 81 | + // 2-2、验证自编号参数，时间参数 | |
| 82 | + String paramClzbh = vehicleDataSyncTask.getParamClzbh(); | |
| 83 | + Date paramFrom = vehicleDataSyncTask.getParamFrom(); | |
| 84 | + Date paramTo = vehicleDataSyncTask.getParamTo(); | |
| 85 | + if (StringUtils.isNotBlank(paramClzbh)) { // 有车辆自编号参数先判断（优先级最高） | |
| 86 | + ktrNamedParams.put("paramClzbh", paramClzbh); | |
| 87 | + ktrNamedParams.put("paramFrom", null); | |
| 88 | + ktrNamedParams.put("paramTo", null); | |
| 89 | + } else if (paramFrom != null && paramTo != null) { // 其次是时间参数 | |
| 90 | + ktrNamedParams.put("paramClzbh", null); | |
| 91 | + ktrNamedParams.put("paramFrom", DateFormatUtils.format(paramFrom, "yyyy-MM-dd") + " 00:00:00"); | |
| 92 | + ktrNamedParams.put("paramTo", DateFormatUtils.format(paramTo, "yyyy-MM-dd") + " 23:59:59"); | |
| 93 | + } else { | |
| 94 | + throw new ScheduleException("车辆信息同步参数异常，车辆参数为空，时间参数为空或部分为空！"); | |
| 95 | + } | |
| 96 | + // 2-3、其他参数 | |
| 82 | 97 | Path fileDir = Paths.get(VehicleDataSyncTaskThread.getLogbackFilePath().toString(), "datasync", "vehicle"); |
| 83 | 98 | String filePrefix = String.format(KTR_FILE_PREFIX_PATTERN, MDC.get("taskLogKey")); |
| 84 | 99 | ktrNamedParams.put("file_validate_error", Paths.get(fileDir.toString(), | ... | ... |
src/main/java/com/bsth/service/schedule/utils/DataToolsProperties.java
| ... | ... | @@ -125,6 +125,9 @@ public class DataToolsProperties { |
| 125 | 125 | @NotNull |
| 126 | 126 | /** 车辆信息同步 */ |
| 127 | 127 | private String vehicleDatasyncktr; |
| 128 | + @NotNull | |
| 129 | + /** 浦东车辆库车辆数据接口地址 */ | |
| 130 | + private String carsInterface; | |
| 128 | 131 | |
| 129 | 132 | // TODO： |
| 130 | 133 | |
| ... | ... | @@ -391,4 +394,12 @@ public class DataToolsProperties { |
| 391 | 394 | public void setVehicleDatasyncktr(String vehicleDatasyncktr) { |
| 392 | 395 | this.vehicleDatasyncktr = vehicleDatasyncktr; |
| 393 | 396 | } |
| 397 | + | |
| 398 | + public String getCarsInterface() { | |
| 399 | + return carsInterface; | |
| 400 | + } | |
| 401 | + | |
| 402 | + public void setCarsInterface(String carsInterface) { | |
| 403 | + this.carsInterface = carsInterface; | |
| 404 | + } | |
| 394 | 405 | } | ... | ... |
src/main/resources/datatools/config-cloud.properties
| ... | ... | @@ -82,4 +82,8 @@ datatools.guideboards_dataoutputktr=/datatools/ktrs/guideboardDataOutput.ktr |
| 82 | 82 | ##--------------------------- 数据同步ktr ------------------------## |
| 83 | 83 | datatools.vehicle_datasyncktr=/datatools/ktrs/vehicleDataSync.ktr |
| 84 | 84 | |
| 85 | +# 车辆库访问url | |
| 86 | +datatools.cars_interface=http://180.166.5.82:8076/cars/getCarsInterface | |
| 87 | +#datatools.cars_interface=http://192.170.101.132:8076/cars/getCarsInterface | |
| 88 | + | |
| 85 | 89 | # TODO： | ... | ... |
src/main/resources/datatools/config-dev.properties
| ... | ... | @@ -82,6 +82,10 @@ datatools.guideboards_dataoutputktr=/datatools/ktrs/guideboardDataOutput.ktr |
| 82 | 82 | ##--------------------------- 数据同步ktr ------------------------## |
| 83 | 83 | datatools.vehicle_datasyncktr=/datatools/ktrs/vehicleDataSync.ktr |
| 84 | 84 | |
| 85 | +# 车辆库访问url | |
| 86 | +datatools.cars_interface=http://180.166.5.82:8076/cars/getCarsInterface | |
| 87 | +#datatools.cars_interface=http://192.170.101.132:8076/cars/getCarsInterface | |
| 88 | + | |
| 85 | 89 | # TODO： |
| 86 | 90 | |
| 87 | 91 | ... | ... |
src/main/resources/datatools/config-prod.properties
| ... | ... | @@ -82,4 +82,8 @@ datatools.guideboards_dataoutputktr=/datatools/ktrs/guideboardDataOutput.ktr |
| 82 | 82 | ##--------------------------- 数据同步ktr ------------------------## |
| 83 | 83 | datatools.vehicle_datasyncktr=/datatools/ktrs/vehicleDataSync.ktr |
| 84 | 84 | |
| 85 | +# 车辆库访问url | |
| 86 | +datatools.cars_interface=http://180.166.5.82:8076/cars/getCarsInterface | |
| 87 | +#datatools.cars_interface=http://192.170.101.132:8076/cars/getCarsInterface | |
| 88 | + | |
| 85 | 89 | # TODO： | ... | ... |
src/main/resources/datatools/config-test.properties
| ... | ... | @@ -82,4 +82,8 @@ datatools.guideboards_dataoutputktr=/datatools/ktrs/guideboardDataOutput.ktr |
| 82 | 82 | ##--------------------------- 数据同步ktr ------------------------## |
| 83 | 83 | datatools.vehicle_datasyncktr=/datatools/ktrs/vehicleDataSync.ktr |
| 84 | 84 | |
| 85 | +# 车辆库访问url | |
| 86 | +datatools.cars_interface=http://180.166.5.82:8076/cars/getCarsInterface | |
| 87 | +#datatools.cars_interface=http://192.170.101.132:8076/cars/getCarsInterface | |
| 88 | + | |
| 85 | 89 | # TODO： | ... | ... |
src/main/resources/datatools/ktrs/vehicleDataSync.ktr
| ... | ... | @@ -125,11 +125,11 @@ |
| 125 | 125 | </info> |
| 126 | 126 | <notepads> |
| 127 | 127 | <notepad> |
| 128 | - <note>1、json输入step处理太慢，使用javascript模拟代替
2、待处理数据xls输出中已经包含了远程和本地的对比车辆数据（不需要输出整个车辆基础信息xls），
 如果以后添加更多的更新字段，这个步骤要改的
3、注意：命名参数paramClzbh的默认值不要写，否则在进行时间范围同步的时候永远按照paramClzbh默认值同步了，
 TODO：之后会考虑使用新的命名参数paramType区别不同的同步类型</note> | |
| 128 | + <note>1、json输入step处理太慢，使用javascript模拟代替
2、待处理数据xls输出中已经包含了远程和本地的对比车辆数据（不需要输出整个车辆基础信息xls），
 如果以后添加更多的更新字段，这个步骤要改的
3、注意：命名参数paramClzbh的默认值不要写，否则在进行时间范围同步的时候永远按照paramClzbh默认值同步了，
 TODO：之后会考虑使用新的命名参数paramType区别不同的同步类型
4、已经在程序中判断参数的问题了，第3点可以忽略了</note> | |
| 129 | 129 | <xloc>463</xloc> |
| 130 | 130 | <yloc>36</yloc> |
| 131 | - <width>1246</width> | |
| 132 | - <heigth>130</heigth> | |
| 131 | + <width>649</width> | |
| 132 | + <heigth>106</heigth> | |
| 133 | 133 | <fontname>YaHei Consolas Hybrid</fontname> |
| 134 | 134 | <fontsize>12</fontsize> |
| 135 | 135 | <fontbold>N</fontbold> |
| ... | ... | @@ -727,6 +727,30 @@ |
| 727 | 727 | </attributes> |
| 728 | 728 | </connection> |
| 729 | 729 | <connection> |
| 730 | + <name>外网杨高机务oracle</name> | |
| 731 | + <server>58.247.254.118</server> | |
| 732 | + <type>ORACLE</type> | |
| 733 | + <access>Native</access> | |
| 734 | + <database>orcl</database> | |
| 735 | + <port>15211</port> | |
| 736 | + <username>ygjw</username> | |
| 737 | + <password>Encrypted 2be98afc86aa7f2e4cb79ce10c795a5cd</password> | |
| 738 | + <servername/> | |
| 739 | + <data_tablespace/> | |
| 740 | + <index_tablespace/> | |
| 741 | + <attributes> | |
| 742 | + <attribute><code>FORCE_IDENTIFIERS_TO_LOWERCASE</code><attribute>N</attribute></attribute> | |
| 743 | + <attribute><code>FORCE_IDENTIFIERS_TO_UPPERCASE</code><attribute>N</attribute></attribute> | |
| 744 | + <attribute><code>IS_CLUSTERED</code><attribute>N</attribute></attribute> | |
| 745 | + <attribute><code>PORT_NUMBER</code><attribute>15211</attribute></attribute> | |
| 746 | + <attribute><code>PRESERVE_RESERVED_WORD_CASE</code><attribute>N</attribute></attribute> | |
| 747 | + <attribute><code>QUOTE_ALL_FIELDS</code><attribute>N</attribute></attribute> | |
| 748 | + <attribute><code>SUPPORTS_BOOLEAN_DATA_TYPE</code><attribute>Y</attribute></attribute> | |
| 749 | + <attribute><code>SUPPORTS_TIMESTAMP_DATA_TYPE</code><attribute>Y</attribute></attribute> | |
| 750 | + <attribute><code>USE_POOLING</code><attribute>N</attribute></attribute> | |
| 751 | + </attributes> | |
| 752 | + </connection> | |
| 753 | + <connection> | |
| 730 | 754 | <name>外网金高机务oracle</name> |
| 731 | 755 | <server>58.247.254.118</server> |
| 732 | 756 | <type>ORACLE</type> |
| ... | ... | @@ -914,6 +938,32 @@ |
| 914 | 938 | </step> |
| 915 | 939 | |
| 916 | 940 | <step> |
| 941 | + <name>公司，分公司名称代码数据</name> | |
| 942 | + <type>TableInput</type> | |
| 943 | + <description/> | |
| 944 | + <distribute>N</distribute> | |
| 945 | + <custom_distribution/> | |
| 946 | + <copies>1</copies> | |
| 947 | + <partitioning> | |
| 948 | + <method>none</method> | |
| 949 | + <schema_name/> | |
| 950 | + </partitioning> | |
| 951 | + <connection>control_jndi</connection> | |
| 952 | + <sql>select gs.gsmc as gsmc
, fgs.fgsmc as fgsmc
, concat (gs.gsmc, '-', fgs.fgsmc) as allmc
, gs.gsdm as gsdm
, fgs.fgsdm as fgsdm
from
(select business_name as fgsmc
, business_code as fgsdm
, up_code gsdm 
from bsth_c_business 
where up_code in (
select business_code from bsth_c_business where up_code = '88')
) fgs left join
(select business_name as gsmc
, business_code as gsdm
from bsth_c_business
where up_code = '88'
) gs on fgs.gsdm = gs.gsdm</sql> | |
| 953 | + <limit>0</limit> | |
| 954 | + <lookup/> | |
| 955 | + <execute_each_row>N</execute_each_row> | |
| 956 | + <variables_active>N</variables_active> | |
| 957 | + <lazy_conversion_active>N</lazy_conversion_active> | |
| 958 | + <cluster_schema/> | |
| 959 | + <remotesteps> <input> </input> <output> </output> </remotesteps> <GUI> | |
| 960 | + <xloc>176</xloc> | |
| 961 | + <yloc>220</yloc> | |
| 962 | + <draw>Y</draw> | |
| 963 | + </GUI> | |
| 964 | + </step> | |
| 965 | + | |
| 966 | + <step> | |
| 917 | 967 | <name>写日志</name> |
| 918 | 968 | <type>WriteToLog</type> |
| 919 | 969 | <description/> |
| ... | ... | @@ -3012,32 +3062,6 @@ |
| 3012 | 3062 | </GUI> |
| 3013 | 3063 | </step> |
| 3014 | 3064 | |
| 3015 | - <step> | |
| 3016 | - <name>公司，分公司名称代码数据</name> | |
| 3017 | - <type>TableInput</type> | |
| 3018 | - <description/> | |
| 3019 | - <distribute>N</distribute> | |
| 3020 | - <custom_distribution/> | |
| 3021 | - <copies>1</copies> | |
| 3022 | - <partitioning> | |
| 3023 | - <method>none</method> | |
| 3024 | - <schema_name/> | |
| 3025 | - </partitioning> | |
| 3026 | - <connection>control_jndi</connection> | |
| 3027 | - <sql>select gs.gsmc as gsmc
, fgs.fgsmc as fgsmc
, concat (gs.gsmc, '-', fgs.fgsmc) as allmc
, gs.gsdm as gsdm
, fgs.fgsdm as fgsdm
from
(select business_name as fgsmc
, business_code as fgsdm
, up_code gsdm 
from bsth_c_business 
where up_code in (
select business_code from bsth_c_business where up_code = '88')
) fgs left join
(select business_name as gsmc
, business_code as gsdm
from bsth_c_business
where up_code = '88'
) gs on fgs.gsdm = gs.gsdm</sql> | |
| 3028 | - <limit>0</limit> | |
| 3029 | - <lookup/> | |
| 3030 | - <execute_each_row>N</execute_each_row> | |
| 3031 | - <variables_active>N</variables_active> | |
| 3032 | - <lazy_conversion_active>N</lazy_conversion_active> | |
| 3033 | - <cluster_schema/> | |
| 3034 | - <remotesteps> <input> </input> <output> </output> </remotesteps> <GUI> | |
| 3035 | - <xloc>176</xloc> | |
| 3036 | - <yloc>220</yloc> | |
| 3037 | - <draw>Y</draw> | |
| 3038 | - </GUI> | |
| 3039 | - </step> | |
| 3040 | - | |
| 3041 | 3065 | <step_error_handling> |
| 3042 | 3066 | <error> |
| 3043 | 3067 | <source_step>JavaScript解析json生成数据</source_step> | ... | ... |