Flink sql 集成hive metastore对接s3存储

櫰木2年前 (2023-12-25)技术文章1493

1、hive metastore使用s3如何适配

1、在flink conf目录中添加core-site.xml和hive-site.xml

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
    <property>
        <name>fs.s3a.connection.ssl.enabled</name>
        <value>true</value>
    </property>
    <property>
        <name>fs.defaultFS</name>
        <value>s3a://spark1125</value>
    </property>
    <property>
        <name>fs.s3a.path.style.access</name>
        <value>true</value>
    </property>
    <property>
        <name>fs.s3a.access.key</name>
        <value>admin</value>
    </property>
    <property>
        <name>fs.s3a.secret.key</name>
        <value>Admin@123</value>
    </property>
    <property>
        <name>fs.s3a.impl</name>
        <value>org.apache.hadoop.fs.s3a.S3AFileSystem</value>
    </property>
    <property>
        <name>fs.s3a.endpoint</name>
        <value>http://172.16.104.165:30153</value>
    </property>
    <property>
        <name>fs.s3a.fast.upload</name>
        <value>true</value>
    </property>
    <property>
        <name>fs.s3a.multipart.threshold</name>
        <value>512M</value>
    </property>
    <property>
        <name>fs.s3a.socket.recv.buffer</name>
        <value>65536</value>
    </property>
    <property>
        <name>fs.s3a.socket.send.buffer</name>
        <value>65536</value>
    </property>
    <property>
        <name>fs.s3a.fast.upload.active.blocks</name>
        <value>2048</value>
    </property>
    <property>
        <name>fs.s3a.fast.upload.buffer</name>
        <value>disk</value>
    </property>
    <property>
        <name>fs.s3a.max.total.tasks</name>
        <value>2048</value>
    </property>
    <property>
        <name>fs.s3a.threads.max</name>
        <value>2048</value>
    </property>
    <property>
        <name>fs.s3a.committer.staging.conflict-mode</name>
        <value>append</value>
    </property>
    <property>
        <name>fs.s3a.connection.establish.timeout</name>
        <value>5000</value>
    </property>
    <property>
        <name>fs.s3a.multipart.size</name>
        <value>512M</value>
    </property>
    <property>
        <name>fs.s3a.committer.magic.enabled</name>
        <value>true</value>
    </property>
    <property>
        <name>fs.s3a.committer.abort.pending.uploads</name>
        <value>true</value>
    </property>
    <property>
        <name>fs.s3a.committer.threads</name>
        <value>8</value>
    </property>
    <property>
        <name>fs.s3a.committer.generate.uuid</name>
        <value>false</value>
    </property>
    <property>
        <name>fs.s3a.committer.require.uuid</name>
        <value>false</value>
    </property>
    <property>
        <name>fs.s3a.committer.name</name>
        <value>magic</value>
    </property>
</configuration>

<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
    <property>
        <name>hive.metastore.event.db.notification.api.auth</name>
        <value>false</value>
    </property>
    <property>
        <name>hive.metastore.uris</name>
        <value>thrift://172.16.104.165:31228</value>
    </property>
    <property>
        <name>javax.jdo.option.ConnectionURL</name>
        <value>jdbc:mysql://172.16.104.165:30066/spark1125?useSSL=false</value>
    </property>
    <property>
        <name>javax.jdo.option.ConnectionDriverName</name>
        <value>com.mysql.cj.jdbc.Driver</value>
    </property>
    <property>
        <name>javax.jdo.option.ConnectionUserName</name>
        <value>root</value>
    </property>
    <property>
        <name>javax.jdo.option.ConnectionPassword</name>
        <value>123456</value>
    </property>
    <property>
        <name>hive.metastore.warehouse.dir</name>
        <value>s3a://spark1125/hive/warehouse</value>
    </property>
</configuration>

返回列表

上一篇：Flink sql 集成hive metastore-测试

下一篇：Elasticsearch数据生命周期如何规划