Commit e294f93b authored by 周昊's avatar 周昊
Browse files

Initial commit

parents

dependency-reduced-pom.xml

0 → 100644
+150 −0
Original line number Diff line number Diff line 
<?xml version="1.0" encoding="UTF-8"?>

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">

  <modelVersion>4.0.0</modelVersion>

  <groupId>my-flink-project</groupId>

  <artifactId>my-flink-project</artifactId>

  <name>Flink Quickstart Job</name>

  <version>0.1</version>

  <url>http://www.myorganization.org</url>

  <build>

    <resources>

      <resource>

        <directory>src/main/java</directory>

        <includes>

          <include>**/*.properties</include>

          <include>**/*.xml</include>

        </includes>

      </resource>

      <resource>

        <directory>src/main/resources</directory>

        <includes>

          <include>**/*.properties</include>

          <include>**/*.xml</include>

        </includes>

      </resource>

    </resources>

    <pluginManagement>

      <plugins>

        <plugin>

          <groupId>org.eclipse.m2e</groupId>

          <artifactId>lifecycle-mapping</artifactId>

          <version>1.0.0</version>

          <configuration>

            <lifecycleMappingMetadata>

              <pluginExecutions>

                <pluginExecution>

                  <pluginExecutionFilter>

                    <groupId>org.apache.maven.plugins</groupId>

                    <artifactId>maven-shade-plugin</artifactId>

                    <versionRange>[3.0.0,)</versionRange>

                    <goals>

                      <goal>shade</goal>

                    </goals>

                  </pluginExecutionFilter>

                  <action>

                    <ignore />

                  </action>

                </pluginExecution>

                <pluginExecution>

                  <pluginExecutionFilter>

                    <groupId>org.apache.maven.plugins</groupId>

                    <artifactId>maven-compiler-plugin</artifactId>

                    <versionRange>[3.1,)</versionRange>

                    <goals>

                      <goal>testCompile</goal>

                      <goal>compile</goal>

                    </goals>

                  </pluginExecutionFilter>

                  <action>

                    <ignore />

                  </action>

                </pluginExecution>

              </pluginExecutions>

            </lifecycleMappingMetadata>

          </configuration>

        </plugin>

      </plugins>

    </pluginManagement>

    <plugins>

      <plugin>

        <artifactId>maven-compiler-plugin</artifactId>

        <version>3.1</version>

        <configuration>

          <source>${java.version}</source>

          <target>${java.version}</target>

        </configuration>

      </plugin>

      <plugin>

        <artifactId>maven-shade-plugin</artifactId>

        <version>3.0.0</version>

        <executions>

          <execution>

            <phase>package</phase>

            <goals>

              <goal>shade</goal>

            </goals>

            <configuration>

              <artifactSet />

              <filters>

                <filter>

                  <artifact>*:*</artifact>

                  <excludes>

                    <exclude>META-INF/*.SF</exclude>

                    <exclude>META-INF/*.DSA</exclude>

                    <exclude>META-INF/*.RSA</exclude>

                  </excludes>

                </filter>

              </filters>

              <transformers>

                <transformer>

                  <mainClass>com.censoft.flink.StreamingJob</mainClass>

                </transformer>

              </transformers>

            </configuration>

          </execution>

        </executions>

      </plugin>

    </plugins>

  </build>

  <profiles>

    <profile>

      <id>add-dependencies-for-IDEA</id>

      <dependencies>

        <dependency>

          <groupId>org.apache.flink</groupId>

          <artifactId>flink-java</artifactId>

          <version>1.9.3</version>

          <scope>compile</scope>

        </dependency>

        <dependency>

          <groupId>org.apache.flink</groupId>

          <artifactId>flink-streaming-java_2.11</artifactId>

          <version>1.9.3</version>

          <scope>compile</scope>

        </dependency>

      </dependencies>

    </profile>

  </profiles>

  <repositories>

    <repository>

      <releases>

        <enabled>false</enabled>

      </releases>

      <snapshots />

      <id>apache.snapshots</id>

      <name>Apache Development Snapshot Repository</name>

      <url>https://repository.apache.org/content/repositories/snapshots/</url>

    </repository>

  </repositories>

  <properties>

    <java.version>1.8</java.version>

    <scala.binary.version>2.11</scala.binary.version>

    <flink.version>1.9.3</flink.version>

    <maven.compiler.source>${java.version}</maven.compiler.source>

    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>

    <maven.compiler.target>${java.version}</maven.compiler.target>

    <fastjson.version>2.0.25</fastjson.version>

    <mqtt.version>1.2.5</mqtt.version>

  </properties>

</project>

pom.xml

0 → 100644
+315 −0
Original line number Diff line number Diff line 
<!--

Licensed to the Apache Software Foundation (ASF) under one

or more contributor license agreements.  See the NOTICE file

distributed with this work for additional information

regarding copyright ownership.  The ASF licenses this file

to you under the Apache License, Version 2.0 (the

"License"); you may not use this file except in compliance

with the License.  You may obtain a copy of the License at



  http://www.apache.org/licenses/LICENSE-2.0



Unless required by applicable law or agreed to in writing,

software distributed under the License is distributed on an

"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY

KIND, either express or implied.  See the License for the

specific language governing permissions and limitations

under the License.

-->

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"

	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">

	<modelVersion>4.0.0</modelVersion>



	<groupId>my-flink-project</groupId>

	<artifactId>my-flink-project</artifactId>

	<version>0.1</version>

	<packaging>jar</packaging>



	<name>Flink Quickstart Job</name>

	<url>http://www.myorganization.org</url>



	<properties>

		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>

		<scala.binary.version>2.11</scala.binary.version>

		<maven.compiler.source>${java.version}</maven.compiler.source>

		<maven.compiler.target>${java.version}</maven.compiler.target>



		<java.version>1.8</java.version>

		<flink.version>1.9.3</flink.version>

		<mqtt.version>1.2.5</mqtt.version>

		<fastjson.version>2.0.25</fastjson.version>

	</properties>



	<repositories>

		<repository>

			<id>apache.snapshots</id>

			<name>Apache Development Snapshot Repository</name>

			<url>https://repository.apache.org/content/repositories/snapshots/</url>

			<releases>

				<enabled>false</enabled>

			</releases>

			<snapshots>

				<enabled>true</enabled>

			</snapshots>

		</repository>

	</repositories>



	<dependencies>

		<dependency>

			<groupId>org.slf4j</groupId>

			<artifactId>slf4j-log4j12</artifactId>

			<version>1.7.7</version>

		</dependency>

		<dependency>

			<groupId>log4j</groupId>

			<artifactId>log4j</artifactId>

			<version>1.2.17</version>

		</dependency>







		<dependency>

			<groupId>org.apache.flink</groupId>

			<artifactId>flink-clients_2.12</artifactId>

			<version>${flink.version}</version>

		</dependency>

		<dependency>

			<groupId>org.apache.flink</groupId>

			<artifactId>flink-scala_2.12</artifactId>

			<version>${flink.version}</version>

		</dependency>

		<dependency>

			<groupId>org.apache.flink</groupId>

			<artifactId>flink-java</artifactId>

			<version>${flink.version}</version>

		</dependency>

		<dependency>

			<groupId>org.apache.flink</groupId>

			<artifactId>flink-streaming-scala_2.12</artifactId>

			<version>${flink.version}</version>

		</dependency>

		<dependency>

			<groupId>org.apache.flink</groupId>

			<artifactId>flink-streaming-java_2.12</artifactId>

			<version>${flink.version}</version>

		</dependency>

		<dependency>

			<groupId>org.apache.flink</groupId>

			<artifactId>flink-table-api-scala-bridge_2.12</artifactId>

			<version>${flink.version}</version>

		</dependency>

		<dependency>

			<groupId>org.apache.flink</groupId>

			<artifactId>flink-table-api-java-bridge_2.12</artifactId>

			<version>${flink.version}</version>

		</dependency>



		<dependency>

			<groupId>org.apache.flink</groupId>

			<artifactId>flink-connector-kafka_2.12</artifactId>

			<version>${flink.version}</version>

		</dependency>



		<!-- mqtt-->

		<dependency>

			<groupId>org.eclipse.paho</groupId>

			<artifactId>org.eclipse.paho.client.mqttv3</artifactId>

			<version>${mqtt.version}</version>

		</dependency>



		<!-- 阿里JSON解析器 -->

		<dependency>

			<groupId>com.alibaba.fastjson2</groupId>

			<artifactId>fastjson2</artifactId>

			<version>${fastjson.version}</version>

		</dependency>



		<!-- lombok工具 -->

		<dependency>

			<groupId>org.projectlombok</groupId>

			<artifactId>lombok</artifactId>

			<version>RELEASE</version>

			<scope>compile</scope>

		</dependency>



		<!-- https://mvnrepository.com/artifact/cn.hutool/hutool-all -->

		<!-- 小而全的Java类库 -->

		<dependency>

			<groupId>cn.hutool</groupId>

			<artifactId>hutool-all</artifactId>

			<version>5.0.7</version>

		</dependency>





		<!--        引入MyBatis的jar包的依赖-->

		<dependency>

			<groupId>org.mybatis</groupId>

			<artifactId>mybatis</artifactId>

			<version>3.5.6</version>

		</dependency>

		<!--        MySQL驱动jar包的依赖-->

		<dependency>

			<groupId>mysql</groupId>

			<artifactId>mysql-connector-java</artifactId>

			<version>8.0.28</version>

		</dependency>

	</dependencies>



	<build>



		<resources>

			<resource>

				<!--                所在的目录-->

				<directory>src/main/java</directory>

				<includes><!--  这个目录下的所有.xml，.properties文件都会被扫描加载到-->

					<include>**/*.properties</include>

					<include>**/*.xml</include>

				</includes>

				<filtering>false</filtering>

			</resource>

			<resource>

				<!--                所在的目录-->

				<directory>src/main/resources</directory>

				<includes><!--  这个目录下的所有.xml，.properties文件都会被扫描加载到-->

					<include>**/*.properties</include>

					<include>**/*.xml</include>

				</includes>

				<filtering>false</filtering>

			</resource>

		</resources>



		<plugins>



			<!-- Java Compiler -->

			<plugin>

				<groupId>org.apache.maven.plugins</groupId>

				<artifactId>maven-compiler-plugin</artifactId>

				<version>3.1</version>

				<configuration>

					<source>${java.version}</source>

					<target>${java.version}</target>

				</configuration>

			</plugin>



			<!-- We use the maven-shade plugin to create a fat jar that contains all necessary dependencies. -->

			<!-- Change the value of <mainClass>...</mainClass> if your program entry point changes. -->

			<plugin>

				<groupId>org.apache.maven.plugins</groupId>

				<artifactId>maven-shade-plugin</artifactId>

				<version>3.0.0</version>

				<executions>

					<!-- Run shade goal on package phase -->

					<execution>

						<phase>package</phase>

						<goals>

							<goal>shade</goal>

						</goals>

						<configuration>

							<artifactSet>

								<!--<excludes>

									<exclude>org.apache.flink:force-shading</exclude>

									<exclude>com.google.code.findbugs:jsr305</exclude>

									<exclude>org.slf4j:*</exclude>

									<exclude>log4j:*</exclude>

								</excludes>-->

							</artifactSet>

							<filters>

								<filter>

									<!-- Do not copy the signatures in the META-INF folder.

									Otherwise, this might cause SecurityExceptions when using the JAR. -->

									<artifact>*:*</artifact>

									<excludes>

										<exclude>META-INF/*.SF</exclude>

										<exclude>META-INF/*.DSA</exclude>

										<exclude>META-INF/*.RSA</exclude>

									</excludes>

								</filter>

							</filters>

							<transformers>

								<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">

									<mainClass>com.censoft.flink.StreamingJob</mainClass>

								</transformer>

							</transformers>

						</configuration>

					</execution>

				</executions>

			</plugin>

		</plugins>



		<pluginManagement>

			<plugins>



				<!-- This improves the out-of-the-box experience in Eclipse by resolving some warnings. -->

				<plugin>

					<groupId>org.eclipse.m2e</groupId>

					<artifactId>lifecycle-mapping</artifactId>

					<version>1.0.0</version>

					<configuration>

						<lifecycleMappingMetadata>

							<pluginExecutions>

								<pluginExecution>

									<pluginExecutionFilter>

										<groupId>org.apache.maven.plugins</groupId>

										<artifactId>maven-shade-plugin</artifactId>

										<versionRange>[3.0.0,)</versionRange>

										<goals>

											<goal>shade</goal>

										</goals>

									</pluginExecutionFilter>

									<action>

										<ignore/>

									</action>

								</pluginExecution>

								<pluginExecution>

									<pluginExecutionFilter>

										<groupId>org.apache.maven.plugins</groupId>

										<artifactId>maven-compiler-plugin</artifactId>

										<versionRange>[3.1,)</versionRange>

										<goals>

											<goal>testCompile</goal>

											<goal>compile</goal>

										</goals>

									</pluginExecutionFilter>

									<action>

										<ignore/>

									</action>

								</pluginExecution>

							</pluginExecutions>

						</lifecycleMappingMetadata>

					</configuration>

				</plugin>

			</plugins>

		</pluginManagement>

	</build>



	<!-- This profile helps to make things run out of the box in IntelliJ -->

	<!-- Its adds Flink's core classes to the runtime class path. -->

	<!-- Otherwise they are missing in IntelliJ, because the dependency is 'provided' -->

	<profiles>

		<profile>

			<id>add-dependencies-for-IDEA</id>



			<activation>

				<property>

					<name>idea.version</name>

				</property>

			</activation>



			<dependencies>

				<dependency>

					<groupId>org.apache.flink</groupId>

					<artifactId>flink-java</artifactId>

					<version>${flink.version}</version>

					<scope>compile</scope>

				</dependency>

				<dependency>

					<groupId>org.apache.flink</groupId>

					<artifactId>flink-streaming-java_${scala.binary.version}</artifactId>

					<version>${flink.version}</version>

					<scope>compile</scope>

				</dependency>

			</dependencies>

		</profile>

	</profiles>



</project>

src/main/java/META-INF/MANIFEST.MF

0 → 100644
+3 −0
Original line number Diff line number Diff line 
Manifest-Version: 1.0

Main-Class: com.censoft.flink.StreamingJob

src/main/java/com/censoft/flink/StreamingJob.java

0 → 100644
+81 −0
Original line number Diff line number Diff line 
package com.censoft.flink;



import com.censoft.flink.domain.AlgorithmPushDto;

import com.censoft.flink.domain.AlgorithmSceneBasePo;

import com.censoft.flink.domain.AlgorithmScenePiecePo;

import com.censoft.flink.mapper.SqlFactory;

import com.censoft.flink.mqtt.MqttConsumer;

import com.censoft.flink.transform.AlgorithmBaseFilterFunction;

import com.censoft.flink.transform.AlgorithmTypeFlatMapFunction;

import com.censoft.flink.transform.ResultExistFilterFunction;

import org.apache.flink.api.common.serialization.SimpleStringSchema;

import org.apache.flink.api.java.utils.ParameterTool;

import org.apache.flink.core.fs.FileSystem;

import org.apache.flink.streaming.api.datastream.DataStream;

import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;

import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;

import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer;



import java.lang.reflect.Method;

import java.util.Arrays;

import java.util.List;





public class StreamingJob {



    public static void main(String[] args) throws Exception {

        //1.获取对应场景id

        ParameterTool parameterTool = ParameterTool.fromArgs(args);

        Long sceneId = parameterTool.getLong("sceneId");



        //2.获取对应算法参数

        //2.1 获取场景数据

        SqlFactory sqlFactory = new SqlFactory();

        AlgorithmSceneBasePo algorithmSceneBasePo = sqlFactory.getAlgorithmSceneBasePo(sceneId);



        //2.2 获取场景算法块数据

        List<AlgorithmScenePiecePo> algorithmScenePieceList = sqlFactory.getAlgorithmScenePieceList(sceneId);





        //1、创建流式执行环境

        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

        env.setParallelism(1);





        //2、接收消息并将数据转化为流

        //todo 根据场景id 拼接对应的mqtt频道名称

        DataStream<AlgorithmPushDto> mqttStream = env.addSource(new MqttConsumer("/censoft/test/" + sceneId));



        //3、进行处理

        //3、1 默认 是否存在结果集判断

        SingleOutputStreamOperator<AlgorithmPushDto> streamOperator = mqttStream

                .filter(new ResultExistFilterFunction());



        //3、2 默认 根据分类,分解多个推送信息

        List<String> algorithmTypes = Arrays.asList(algorithmSceneBasePo.getAlarmTypes().split(","));

        streamOperator = streamOperator.flatMap(new AlgorithmTypeFlatMapFunction(algorithmTypes));





        //3、3 根据配置算法流程 反射对应对象

        for (AlgorithmScenePiecePo algorithmScenePiecePo : algorithmScenePieceList) {

            //通过构造函数 获取算法块对象

            Class<?> filterClazz = Class.forName(algorithmScenePiecePo.getClazz());

            Method method = filterClazz.getMethod("getFilterFunction", AlgorithmSceneBasePo.class, AlgorithmScenePiecePo.class);

            AlgorithmBaseFilterFunction filterFunction = (AlgorithmBaseFilterFunction) method.invoke(null, algorithmSceneBasePo, algorithmScenePiecePo);

            streamOperator = streamOperator.filter(filterFunction);

        }



        //3、4 Object -> String

        SingleOutputStreamOperator<String> outputStreamOperator = streamOperator.map(algorithmPushDto -> algorithmPushDto.toString());



        //3、5 输出kafka



        outputStreamOperator.addSink(new FlinkKafkaProducer("192.168.10.137:9092", "test-topic", new SimpleStringSchema()));



        //3、6 打印输出

        outputStreamOperator.print();

        outputStreamOperator.writeAsText("D:/word1.txt", FileSystem.WriteMode.OVERWRITE).setParallelism(1);

        //3、7 自动执行

        env.execute();

    }

}

src/main/java/com/censoft/flink/domain/AlgorithmFrameNumberDto.java

0 → 100644
+18 −0
Original line number Diff line number Diff line 
package com.censoft.flink.domain;



import lombok.Data;



/**

 * 帧算法缓存对象

 */

@Data

public class AlgorithmFrameNumberDto {



    //上次计数时间戳

    private Long timeStamp;



    //累计计数数量

    private Integer frameNumber;





}