CONA BLOG

# gcc가 설치되어있나 확인
$ gcc --version

# 없으면 패키지 매니저 yum을 통해 install
$ yum install -y gcc

# wget으로 perl 다운로드 후 압축 해제
$ wget http://www.cpan.org/src/5.0/perl-5.24.1.tar.gz
$ tar xvf perl-5.24.1.tar.gz
$ cd perl-5.24.1

$ ./Configure -des -Dprefix=$HOME/localperl

$ yum install make

$ make test
$ make install

Java 8 설치

# centOS pakage manager 업데이트
$ yum -y update

# java 8 install
$ yum install java-1.8.0-openjdk

# 설치가 잘 되었는지 확인
$ java -version

 Java path 설정

# java 위치 확인
$ which javac
$ readlink -f /usr/bin/javac

# 출력된 결과에서 /bin/javac 떼고 나머지 경로 복사

$ update-alternatives --config java

# profile에 export JAVA_HOME에 아까 복사한 경로 추가
$ vi .bash_profile
// export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.191.b12-1.el7_6.x86_64

# 수정된 bash_profile 적용
$ source .bash_profile
$ echo $JAVA_HOME

Download Druid 0.13.0

# wget으로 다운로드
$ wget https://archive.apache.org/dist/incubator/druid/0.13.0-incubating/apache-druid-0.13.0-incubating-bin.tar.gz

# 압축해제
$ tar -xzf apache-druid-0.13.0-incubating-bin.tar.gz

# 폴더로 이동
$ cd apache-druid-0.13.0-incubating

Download Zookeeper 3.4.11

# curl로 zookeeper 다운로드 3.4.11 버전
$ curl https://archive.apache.org/dist/zookeeper/zookeeper-3.4.11/zookeeper-3.4.11.tar.gz -o zookeeper-3.4.11.tar.gz

# 압축해제
$ tar -xzf zookeeper-3.4.11.tar.gz

# zookeeper 이동
$ mv zookeeper-3.4.11 zk

Druid 실행

# From the apache-druid-0.13.0-incubating package root, run the following command
$ bin/supervise -c quickstart/tutorial/conf/tutorial-cluster.conf

참고

https://druid.apache.org/docs//0.13.0-incubating/tutorials/tutorial-kafka.html

# 1d = 86400000ms
$ ./bin/kafka-topics.sh --zookeeper <Zookeeper ip address> --alter --topic \
<Topic name> --config retention.ms=86400000

# 변경되었는지 확인 retention.ms=86400000면 성공
$ ./bin/kafka-topics.sh  --describe --zookeeper <Zookeeper ip address> --topic <Topic name>

참고

https://knight76.tistory.com/entry/kafka%EC%97%90%EC%84%9C-retentionms-%EC%88%98%EC%A0%95%ED%95%98%EB%8A%94-%EB%B0%A9%EB%B2%95

https://github.com/occidere/TIL/issues/13

$ wget http://mirror.navercorp.com/apache/kafka/2.4.0/kafka_2.12-2.4.0.tgz
$ tar -xzf kafka_2.12-2.4.0.tgz
$ cd kafka_2.12-2.4.0


# kafka 폴더 안에서 zookeeper 실행
$ bin/zookeeper-server-start.sh config/zookeeper.properties

# kafka 실행
$ bin/kafka-server-start.sh config/server.properties


# create topic
$ bin/kafka-topics.sh --create --bootstrap-server localhost:9092 \
--replication-factor 1 --partitions 1 --topic <topic name>

# topic 리스트 확인
$ bin/kafka-topics.sh --list --bootstrap-server localhost:9092

참고

https://kafka.apache.org/quickstart

https://soyoung-new-challenge.tistory.com/61

https://sejoung.github.io/2018/09/2018-09-11-kafka_quickstart/#7%EB%8B%A8%EA%B3%84-Use-Kafka-Connect-to-import-export-data

https://12bme.tistory.com/521

yum으로 python dependency 설치

$ sudo yum -y update
$ sudo yum -y groupinstall "Development Tools"
$ sudo yum -y install openssl-devel bzip2-devel libffi-devel

# gcc 설치 잘되었는지 확인
$ gcc --version

Python3.7 download

# 링크에서 다운로드
$ wget https://www.python.org/ftp/python/3.7.4/Python-3.7.4.tgz

# 압축해제
$ tar xvfz Python-3.7.4.tgz 

$ cd Python-3.7.4


$ ./configure --enable-optimizations
$ sudo make altinstall

# python 버전확인, 설치확인
$ python --version

./configure 에서 오류 발생

configure: error: no acceptable c compiler found in $path

해결방법

-> gcc 설치 후 다시 ./configure부터 시작

$ sudo yum install gcc

구축환경

Ubuntu 18.04

python 3.7

# pymysql 설치
$ pip3 install pymysql


# confluent-kafka 설치
$ pip3 install confluent-kafka