solr and atlas.md 9.9 KB
[TOC]
1、solr的安装与使用
1.1、solr的安装---（三台机器都需要安装solr）
# 1.将安装包放入/opt/modules目录下
solr.zip
# 2.解压缩
unzip solr.zip
# 3.创建用户
sudo useradd solr
echo solr | sudo passwd --stdin solr
# 4.修改 solr 目录的所有者为 solr 用户
sudo chown -R solr:solr /opt/modules/solr
# 5.修改/opt/modules/solr/bin/solr.in.sh 文件中的以下属性
vim /opt/modules/solr/bin/solr.in.sh
ZK_HOST="lab1:2181,lab2:2181,lab3:2181"
SOLR_HOST="自己的ip"
1.2、启动集群
# 1.启动solr
ssh lab1 sudo -i -u solr /opt/modules/solr/bin/solr start # 关闭为stop
ssh lab2 sudo -i -u solr /opt/modules/solr/bin/solr start
ssh lab3 sudo -i -u solr /opt/modules/solr/bin/solr start
sudo -i -u solr /opt/modules/solr/bin/solr start -c -p 8983 -z 150.158.138.99:2181,49.235.67.21:2181,42.192.195.253:2181 -force
# 强制启动
./solr start -c -p 8983 -z lab1:2181,lab2:2181,lab3:2181 -force
# 2.创建collection---在一个节点上面运行即可
sudo -i -u solr /opt/modules/solr/bin/solr create -c vertex_index -d /opt/modules/atlas-2.2.0/conf/solr -shards 3 -replicationFactor 1
sudo -i -u solr /opt/modules/solr/bin/solr create -c edge_index -d /opt/modules/atlas-2.2.0/conf/solr -shards 3 -replicationFactor 1
sudo -i -u solr /opt/modules/solr/bin/solr create -c fulltext_index -d /opt/modules/atlas-2.2.0/conf/solr -shards 3 -replicationFactor 1
# 3.删除collection
curl "http://127.0.0.1:8983/solr/admin/collections?action=DELETE&name=edge_index"
curl "http://127.0.0.1:8983/solr/admin/collections?action=DELETE&name=vertex_index"
curl "http://127.0.0.1:8983/solr/admin/collections?action=DELETE&name=fulltext_index"
2、atlas的安装---（单节点，这里选择二号虚拟机）
2.1、atlas安装
# 1.将安装包放入/opt/modules目录下
atlas-2.2.0.zip
# 2.解压缩
unzip atlas-2.2.0.zip
# 3.修改atlas的配置文件
vim /opt/modules/atlas-2.2.0/conf/atlas-application.properties
# 将其中的lab1，lab2，lab3全部更换为自己的ip，并修改这一段
atlas.rest.address=http://lab2:21000 # 选择几号机器，就写那个IP
# 4.删除原来的hbase的安装包
rm -rf /opt/modules/hbase-2.2.6/lib/commons-configuration-1.6.jar
# 5.移动高版本的安装包
mv commons-configuration-1.10.jar /opt/modules/hbase-2.2.6/lib/commons-configuration-1.10.jar
2.2、atlas集成Hbase
# 拷贝atlas配置文件到hbase的conf中
cp atlas-application.properties /opt/modules/hbase-2.2.6/conf/
3、Altas数据恢复
Altas数据恢复涉及solr以及两张hbase表——apache_atlas_entity_audit、apache_atlas_janus。
3.1、hbase表数据导出和恢复
在
hdfs中新建文件夹hadoop fs -mkdir /tmp/altas_data将这两张表的数据导出到
hdfs中hbase org.apache.hadoop.hbase.mapreduce.Export apache_atlas_entity_audit hdfs://lab1:8020/tmp/hbase/altas_data/apache_atlas_entity_audit hbase org.apache.hadoop.hbase.mapreduce.Export apache_atlas_janus hdfs://lab1:8020/tmp/hbase/altas_data/apache_atlas_janus新建本地文件夹
altas_data，将导出的文件保存到该文件夹内hadoop fs -get /tmp/hbase/altas_data/apache_atlas_entity_audit ./ hadoop fs -get /tmp/hbase/altas_data/apache_atlas_janus ./将这两份文件上传至目标机器的
hdfs中# 将上述的两个文件夹，放入lab2j hdfs dfs -put ./altas_data/ hdfs://lab1:8020/tmp/查看
hbase表结构并在目标机器中创建这两张表，注意需要删除原表结构中的TTL => 'FOREVER'# 1.创建apache_atlas_janus表 create 'apache_atlas_janus', {NAME => 'e', VERSIONS => '1', EVICT_BLOCKS_ON_CLOSE => 'false', NEW_VERSION_BEHAVIOR => 'false',KEEP_DELETED_CELLS => 'FALSE', CACHE_DATA_ON_WRITE => 'false', DATA_BLOCK_ENCODING => 'NONE', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', CACHE_INDEX_ON_WRITE => 'false', IN_MEMORY => 'false', CACHE_BLOOMS_ON_WRITE => 'false', PREFETCH_BLOCKS_ON_OPEN => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536'}, {NAME => 'f', VERSIONS => '1', EVICT_BLOCKS_ON_CLOSE => 'false', NEW_VERSION_BEHAVIOR => 'false', KEEP_DELETED_CELLS => 'FALSE', CACHE_DATA_ON_WRITE => 'false', DATA_BLOCK_ENCODING => 'NONE', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', CACHE_INDEX_ON_WRITE => 'false', IN_MEMORY => 'false', CACHE_BLOOMS_ON_WRITE => 'false', PREFETCH_BLOCKS_ON_OPEN => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536'} ,{NAME => 'g', VERSIONS => '1', EVICT_BLOCKS_ON_CLOSE => 'false', NEW_VERSION_BEHAVIOR => 'false',KEEP_DELETED_CELLS => 'FALSE', CACHE_DATA_ON_WRITE => 'false', DATA_BLOCK_ENCODING => 'NONE', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', CACHE_INDEX_ON_WRITE => 'false', IN_MEMORY => 'false', CACHE_BLOOMS_ON_WRITE => 'false', PREFETCH_BLOCKS_ON_OPEN => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536'},{NAME => 'h', VERSIONS => '1', EVICT_BLOCKS_ON_CLOSE => 'false', NEW_VERSION_BEHAVIOR => 'false',KEEP_DELETED_CELLS => 'FALSE', CACHE_DATA_ON_WRITE => 'false', DATA_BLOCK_ENCODING => 'NONE', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', CACHE_INDEX_ON_WRITE => 'false', IN_MEMORY => 'false', CACHE_BLOOMS_ON_WRITE => 'false', PREFETCH_BLOCKS_ON_OPEN => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536'} ,{NAME => 'i', VERSIONS => '1', EVICT_BLOCKS_ON_CLOSE => 'false', NEW_VERSION_BEHAVIOR => 'false', KEEP_DELETED_CELLS => 'FALSE', CACHE_DATA_ON_WRITE => 'false', DATA_BLOCK_ENCODING => 'NONE', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', CACHE_INDEX_ON_WRITE => 'false', IN_MEMORY => 'false', CACHE_BLOOMS_ON_WRITE => 'false', PREFETCH_BLOCKS_ON_OPEN => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536'}, {NAME => 'l', VERSIONS => '1', EVICT_BLOCKS_ON_CLOSE => 'false', NEW_VERSION_BEHAVIOR => 'false', KEEP_DELETED_CELLS => 'FALSE', CACHE_DATA_ON_WRITE => 'false', DATA_BLOCK_ENCODING => 'NONE', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', CACHE_INDEX_ON_WRITE => 'false', IN_MEMORY => 'false', CACHE_BLOOMS_ON_WRITE => 'false', PREFETCH_BLOCKS_ON_OPEN => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536'}, {NAME => 'm', VERSIONS => '1', EVICT_BLOCKS_ON_CLOSE => 'false', NEW_VERSION_BEHAVIOR => 'false',KEEP_DELETED_CELLS => 'FALSE', CACHE_DATA_ON_WRITE => 'false', DATA_BLOCK_ENCODING => 'NONE', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', CACHE_INDEX_ON_WRITE => 'false', IN_MEMORY => 'false',CACHE_BLOOMS_ON_WRITE => 'false', PREFETCH_BLOCKS_ON_OPEN => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536'}, {NAME => 's', VERSIONS => '1', EVICT_BLOCKS_ON_CLOSE => 'false', NEW_VERSION_BEHAVIOR => 'false',KEEP_DELETED_CELLS => 'FALSE', CACHE_DATA_ON_WRITE => 'false', DATA_BLOCK_ENCODING => 'NONE', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', CACHE_INDEX_ON_WRITE => 'false', IN_MEMORY => 'false', CACHE_BLOOMS_ON_WRITE => 'false', PREFETCH_BLOCKS_ON_OPEN => 'false', COMPRESSION => 'GZ', BLOCKCACHE => 'true', BLOCKSIZE => '65536'}, {NAME => 't', VERSIONS => '1', EVICT_BLOCKS_ON_CLOSE => 'false', NEW_VERSION_BEHAVIOR => 'false', KEEP_DELETED_CELLS => 'FALSE', CACHE_DATA_ON_WRITE => 'false', DATA_BLOCK_ENCODING => 'NONE', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', CACHE_INDEX_ON_WRITE => 'false', IN_MEMORY => 'false', CACHE_BLOOMS_ON_WRITE => 'false', PREFETCH_BLOCKS_ON_OPEN => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536'} # 2.创建apache_atlas_entity_audit create 'apache_atlas_entity_audit', {NAME => 'dt', VERSIONS => '1', EVICT_BLOCKS_ON_CLOSE => 'false',NEW_VERSION_BEHAVIOR => 'false', KEEP_DELETED_CELLS => 'FALSE', CACHE_DATA_ON_WRITE => 'false', DATA_BLOCK_ENCODING => 'FAST_DIFF', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', CACHE_INDEX_ON_WRITE => 'false', IN_MEMORY => 'false', CACHE_BLOOMS_ON_WRITE => 'false',PREFETCH_BLOCKS_ON_OPEN => 'false', COMPRESSION => 'GZ', BLOCKCACHE => 'true', BLOCKSIZE => '65536'}将导出的数据导入到新建的
hbase表中hbase org.apache.hadoop.hbase.mapreduce.Import apache_atlas_janus hdfs://lab1:8020/tmp/atlas_data/apache_atlas_entity_audit hbase org.apache.hadoop.hbase.mapreduce.Import apache_atlas_janus hdfs://lab1:8020/tmp/atlas_data/apache_atlas_janus
3.2、solr数据的恢复--- (单节点执行，这里选择lab2，即上面atlas的虚拟机)
# 0.
mkdir -p /opt/modules/solr/backup
# 移动备份文件到这个文件夹
mv ./* /opt/modules/solr/backup/
# 给solr权限
chown -R solr:solr /opt/modules/solr/backup
# 1.创建备份,一下操作皆为solr用户
su solr
curl 'http://127.0.0.1:8983/solr/fulltext_index/replication?command=backup&location=/opt/modules/solr/backup&name=fulltext_index.bak'
curl 'http://127.0.0.1:8983/solr/vertex_index/replication?command=backup&location=/opt/modules/solr/backup&name=vertex_index.bak'
curl 'http://127.0.0.1:8983/solr/edge_index/replication?command=backup&location=/opt/modules/solr/backup&name=edge_index.bak'
# 2.恢复备份,将备份拷贝到/opt/modules/solr/backup目录下
curl 'http://127.0.0.1:8983/solr/fulltext_index/replication?command=restore&location=/opt/modules/solr/backup&name=fulltext_index.bak'
curl 'http://127.0.0.1:8983/solr/vertex_index/replication?command=restore&location=/opt/modules/solr/backup&name=vertex_index.bak'
curl 'http://127.0.0.1:8983/solr/edge_index/replication?command=restore&location=/opt/modules/solr/backup&name=edge_index.bak'
# 3.查看备份细节
curl "http://localhost:8983/solr/fulltext_index/replication?command=details"
3.3、启动atlas
# 进入atlas安装目录执行
bin/atlas_start.py
# 稍等10-20分钟访问
http://lab2:21000