flink写入starrocks案例_starrocksink

StarRocks Connector

		<dependency>
            <groupId>mysql</groupId>
            <artifactId>mysql-connector-java</artifactId>
            <version>8.0.20</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-connector-jdbc_2.12</artifactId>
            <version>1.13.5</version>
        </dependency>

        <dependency>
            <groupId>com.starrocks</groupId>
            <artifactId>flink-connector-starrocks</artifactId>
            <version>1.1.16_flink-1.13_2.12</version>
        </dependency>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16

StarRocksSinkExample

import com.starrocks.connector.flink.StarRocksSink;
import com.starrocks.connector.flink.table.StarRocksSinkOptions;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.table.api.DataTypes;
import org.apache.flink.table.api.TableSchema;

public class StarRocksSinkExample {

    static class Book {
        public Book(Long id, String title, String authors, Integer year) {
            this.id = id;
            this.title = title;
            this.authors = authors;
            this.year = year;
        }
        final Long id;
        final String title;
        final String authors;
        final Integer year;
    }

    public static void main(String[] args) throws Exception {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

        env.fromElements(
                new Book(101L, "Stream Processing with Apache Flink", "Fabian Hueske, Vasiliki Kalavri", 2019),
                new Book(102L, "Streaming Systems", "Tyler Akidau, Slava Chernyak, Reuven Lax", 2018),
                new Book(103L, "Designing Data-Intensive Applications", "Martin Kleppmann", 2017),
                new Book(104L, "Kafka: The Definitive Guide", "Gwen Shapira, Neha Narkhede, Todd Palino", 2017)
        ).addSink(
                StarRocksSink.sink(
                        TableSchema.builder()
                                .field("id", DataTypes.BIGINT())
                                .field("title", DataTypes.VARCHAR(100))
                                .field("authors", DataTypes.VARCHAR(100))
                                .field("year", DataTypes.INT())
                                .build(),
                        StarRocksSinkOptions.builder()
                                .withProperty("connector","starrocks")
                                .withProperty("jdbc-url","jdbc:mysql://10.x.x.x:9030?characterEncoding=utf-8&useSSL=false")
                                .withProperty("load-url","10.x.x.x:8030;10.x.x.x:8030;10.x.x.x:8030")
                                .withProperty("username","root")
                                .withProperty("password","pass")
                                .withProperty("table-name","book")
                                .withProperty("database-name","database_name")
                                .withProperty("sink.properties.column_separator", "\\x01")
                                .withProperty("sink.properties.row_delimiter", "\\x02")
                                .withProperty("sink.buffer-flush.interval-ms", "10000")
                                .build(),
                        (slot,book)->{
                            slot[0] = book.id;
                            slot[1] = book.title;
                            slot[2] = book.authors;
                            slot[3] = book.year;
                        }
                )
        );

        env.execute();
    }
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61