2021-11-02 06:29:58 +00:00
# This test is very comprehensive. It tests whether all hadoop services work well with each other.
# Run this when updating the Hadoop package or making significant changes to the hadoop module.
# For a more basic test, see hdfs.nix and yarn.nix
2021-10-20 20:33:56 +00:00
import ../make-test-python.nix ( { pkgs , . . . }: {
nodes = let
package = pkgs . hadoop ;
coreSite = {
2021-11-02 06:29:58 +00:00
" f s . d e f a u l t F S " = " h d f s : / / n s 1 " ;
} ;
hdfsSite = {
" d f s . n a m e n o d e . r p c - b i n d - h o s t " = " 0 . 0 . 0 . 0 " ;
" d f s . n a m e n o d e . h t t p - b i n d - h o s t " = " 0 . 0 . 0 . 0 " ;
" d f s . n a m e n o d e . s e r v i c e r p c - b i n d - h o s t " = " 0 . 0 . 0 . 0 " ;
# HA Quorum Journal Manager configuration
" d f s . n a m e s e r v i c e s " = " n s 1 " ;
" d f s . h a . n a m e n o d e s . n s 1 " = " n n 1 , n n 2 " ;
" d f s . n a m e n o d e . s h a r e d . e d i t s . d i r . n s 1 . n n 1 " = " q j o u r n a l : / / j n 1 : 8 4 8 5 ; j n 2 : 8 4 8 5 ; j n 3 : 8 4 8 5 / n s 1 " ;
" d f s . n a m e n o d e . s h a r e d . e d i t s . d i r . n s 1 . n n 2 " = " q j o u r n a l : / / j n 1 : 8 4 8 5 ; j n 2 : 8 4 8 5 ; j n 3 : 8 4 8 5 / n s 1 " ;
" d f s . n a m e n o d e . r p c - a d d r e s s . n s 1 . n n 1 " = " n n 1 : 8 0 2 0 " ;
" d f s . n a m e n o d e . r p c - a d d r e s s . n s 1 . n n 2 " = " n n 2 : 8 0 2 0 " ;
" d f s . n a m e n o d e . s e r v i c e r p c - a d d r e s s . n s 1 . n n 1 " = " n n 1 : 8 0 2 2 " ;
" d f s . n a m e n o d e . s e r v i c e r p c - a d d r e s s . n s 1 . n n 2 " = " n n 2 : 8 0 2 2 " ;
" d f s . n a m e n o d e . h t t p - a d d r e s s . n s 1 . n n 1 " = " n n 1 : 9 8 7 0 " ;
" d f s . n a m e n o d e . h t t p - a d d r e s s . n s 1 . n n 2 " = " n n 2 : 9 8 7 0 " ;
# Automatic failover configuration
" d f s . c l i e n t . f a i l o v e r . p r o x y . p r o v i d e r . n s 1 " = " o r g . a p a c h e . h a d o o p . h d f s . s e r v e r . n a m e n o d e . h a . C o n f i g u r e d F a i l o v e r P r o x y P r o v i d e r " ;
" d f s . h a . a u t o m a t i c - f a i l o v e r . e n a b l e d . n s 1 " = " t r u e " ;
" d f s . h a . f e n c i n g . m e t h o d s " = " s h e l l ( t r u e ) " ;
" h a . z o o k e e p e r . q u o r u m " = " z k 1 : 2 1 8 1 " ;
} ;
yarnSiteHA = {
" y a r n . r e s o u r c e m a n a g e r . z k - a d d r e s s " = " z k 1 : 2 1 8 1 " ;
" y a r n . r e s o u r c e m a n a g e r . h a . e n a b l e d " = " t r u e " ;
" y a r n . r e s o u r c e m a n a g e r . h a . r m - i d s " = " r m 1 , r m 2 " ;
" y a r n . r e s o u r c e m a n a g e r . h o s t n a m e . r m 1 " = " r m 1 " ;
" y a r n . r e s o u r c e m a n a g e r . h o s t n a m e . r m 2 " = " r m 2 " ;
" y a r n . r e s o u r c e m a n a g e r . h a . a u t o m a t i c - f a i l o v e r . e n a b l e d " = " t r u e " ;
" y a r n . r e s o u r c e m a n a g e r . c l u s t e r - i d " = " c l u s t e r 1 " ;
# yarn.resourcemanager.webapp.address needs to be defined even though yarn.resourcemanager.hostname is set. This shouldn't be necessary, but there's a bug in
# hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java:70
# that causes AM containers to fail otherwise.
" y a r n . r e s o u r c e m a n a g e r . w e b a p p . a d d r e s s . r m 1 " = " r m 1 : 8 0 8 8 " ;
" y a r n . r e s o u r c e m a n a g e r . w e b a p p . a d d r e s s . r m 2 " = " r m 2 : 8 0 8 8 " ;
2021-10-20 20:33:56 +00:00
} ;
in {
2021-11-02 06:29:58 +00:00
zk1 = { . . . }: {
services . zookeeper . enable = true ;
networking . firewall . allowedTCPPorts = [ 2181 ] ;
} ;
# HDFS cluster
nn1 = { pkgs , options , . . . }: {
2021-10-20 20:33:56 +00:00
services . hadoop = {
2021-11-02 06:29:58 +00:00
inherit package coreSite hdfsSite ;
2022-01-08 10:49:15 +00:00
hdfs . namenode = {
enable = true ;
openFirewall = true ;
} ;
2021-11-02 16:16:48 +00:00
hdfs . zkfc . enable = true ;
2021-11-02 06:29:58 +00:00
} ;
} ;
nn2 = { pkgs , options , . . . }: {
services . hadoop = {
inherit package coreSite hdfsSite ;
2022-01-08 10:49:15 +00:00
hdfs . namenode = {
enable = true ;
openFirewall = true ;
} ;
2021-11-02 16:16:48 +00:00
hdfs . zkfc . enable = true ;
2021-10-20 20:33:56 +00:00
} ;
} ;
2021-11-02 06:29:58 +00:00
jn1 = { pkgs , options , . . . }: {
2021-10-20 20:33:56 +00:00
services . hadoop = {
2021-11-02 06:29:58 +00:00
inherit package coreSite hdfsSite ;
2022-01-08 10:49:15 +00:00
hdfs . journalnode = {
enable = true ;
openFirewall = true ;
} ;
2021-11-02 06:29:58 +00:00
} ;
} ;
jn2 = { pkgs , options , . . . }: {
services . hadoop = {
inherit package coreSite hdfsSite ;
2022-01-08 10:49:15 +00:00
hdfs . journalnode = {
enable = true ;
openFirewall = true ;
} ;
2021-11-02 06:29:58 +00:00
} ;
} ;
jn3 = { pkgs , options , . . . }: {
services . hadoop = {
inherit package coreSite hdfsSite ;
2022-01-08 10:49:15 +00:00
hdfs . journalnode = {
enable = true ;
openFirewall = true ;
} ;
2021-11-02 06:29:58 +00:00
} ;
} ;
dn1 = { pkgs , options , . . . }: {
services . hadoop = {
inherit package coreSite hdfsSite ;
2022-01-08 10:49:15 +00:00
hdfs . datanode = {
enable = true ;
openFirewall = true ;
} ;
2021-10-20 20:33:56 +00:00
} ;
2021-11-02 06:29:58 +00:00
} ;
# YARN cluster
rm1 = { pkgs , options , . . . }: {
services . hadoop = {
inherit package coreSite hdfsSite ;
yarnSite = options . services . hadoop . yarnSite . default // yarnSiteHA ;
2022-01-08 10:49:15 +00:00
yarn . resourcemanager = {
enable = true ;
openFirewall = true ;
} ;
2021-11-02 06:29:58 +00:00
} ;
} ;
rm2 = { pkgs , options , . . . }: {
services . hadoop = {
inherit package coreSite hdfsSite ;
yarnSite = options . services . hadoop . yarnSite . default // yarnSiteHA ;
2022-01-08 10:49:15 +00:00
yarn . resourcemanager = {
enable = true ;
openFirewall = true ;
} ;
2021-11-02 06:29:58 +00:00
} ;
} ;
nm1 = { pkgs , options , . . . }: {
2021-10-20 20:33:56 +00:00
virtualisation . memorySize = 2048 ;
2021-11-02 06:29:58 +00:00
services . hadoop = {
inherit package coreSite hdfsSite ;
yarnSite = options . services . hadoop . yarnSite . default // yarnSiteHA ;
2022-01-08 10:49:15 +00:00
yarn . nodemanager = {
enable = true ;
openFirewall = true ;
} ;
2021-11-02 06:29:58 +00:00
} ;
2021-10-20 20:33:56 +00:00
} ;
} ;
testScript = ''
start_all ( )
2021-11-02 06:29:58 +00:00
#### HDFS tests ####
zk1 . wait_for_unit ( " n e t w o r k . t a r g e t " )
jn1 . wait_for_unit ( " n e t w o r k . t a r g e t " )
jn2 . wait_for_unit ( " n e t w o r k . t a r g e t " )
jn3 . wait_for_unit ( " n e t w o r k . t a r g e t " )
nn1 . wait_for_unit ( " n e t w o r k . t a r g e t " )
nn2 . wait_for_unit ( " n e t w o r k . t a r g e t " )
dn1 . wait_for_unit ( " n e t w o r k . t a r g e t " )
zk1 . wait_for_unit ( " z o o k e e p e r " )
jn1 . wait_for_unit ( " h d f s - j o u r n a l n o d e " )
jn2 . wait_for_unit ( " h d f s - j o u r n a l n o d e " )
jn3 . wait_for_unit ( " h d f s - j o u r n a l n o d e " )
zk1 . wait_for_open_port ( 2181 )
jn1 . wait_for_open_port ( 8480 )
jn1 . wait_for_open_port ( 8485 )
jn2 . wait_for_open_port ( 8480 )
jn2 . wait_for_open_port ( 8485 )
# Namenodes must be stopped before initializing the cluster
nn1 . succeed ( " s y s t e m c t l s t o p h d f s - n a m e n o d e " )
nn2 . succeed ( " s y s t e m c t l s t o p h d f s - n a m e n o d e " )
nn1 . succeed ( " s y s t e m c t l s t o p h d f s - z k f c " )
nn2 . succeed ( " s y s t e m c t l s t o p h d f s - z k f c " )
# Initialize zookeeper for failover controller
nn1 . succeed ( " s u d o - u h d f s h d f s z k f c - f o r m a t Z K 2 > & 1 | s y s t e m d - c a t " )
# Format NN1 and start it
nn1 . succeed ( " s u d o - u h d f s h a d o o p n a m e n o d e - f o r m a t 2 > & 1 | s y s t e m d - c a t " )
nn1 . succeed ( " s y s t e m c t l s t a r t h d f s - n a m e n o d e " )
nn1 . wait_for_open_port ( 9870 )
nn1 . wait_for_open_port ( 8022 )
nn1 . wait_for_open_port ( 8020 )
# Bootstrap NN2 from NN1 and start it
nn2 . succeed ( " s u d o - u h d f s h d f s n a m e n o d e - b o o t s t r a p S t a n d b y 2 > & 1 | s y s t e m d - c a t " )
nn2 . succeed ( " s y s t e m c t l s t a r t h d f s - n a m e n o d e " )
nn2 . wait_for_open_port ( 9870 )
nn2 . wait_for_open_port ( 8022 )
nn2 . wait_for_open_port ( 8020 )
nn1 . succeed ( " n e t s t a t - t u l p n e | s y s t e m d - c a t " )
# Start failover controllers
nn1 . succeed ( " s y s t e m c t l s t a r t h d f s - z k f c " )
nn2 . succeed ( " s y s t e m c t l s t a r t h d f s - z k f c " )
2021-10-20 20:33:56 +00:00
2021-11-02 06:29:58 +00:00
# DN should have started by now, but confirm anyway
dn1 . wait_for_unit ( " h d f s - d a t a n o d e " )
# Print states of namenodes
dn1 . succeed ( " s u d o - u h d f s h d f s h a a d m i n - g e t A l l S e r v i c e S t a t e | s y s t e m d - c a t " )
# Wait for cluster to exit safemode
dn1 . succeed ( " s u d o - u h d f s h d f s d f s a d m i n - s a f e m o d e w a i t " )
dn1 . succeed ( " s u d o - u h d f s h d f s h a a d m i n - g e t A l l S e r v i c e S t a t e | s y s t e m d - c a t " )
# test R/W
dn1 . succeed ( " e c h o t e s t f i l e c o n t e n t s | s u d o - u h d f s h d f s d f s - p u t - / t e s t f i l e " )
assert " t e s t f i l e c o n t e n t s " in dn1 . succeed ( " s u d o - u h d f s h d f s d f s - c a t / t e s t f i l e " )
2021-10-20 20:33:56 +00:00
2021-11-02 06:29:58 +00:00
# Test NN failover
nn1 . succeed ( " s y s t e m c t l s t o p h d f s - n a m e n o d e " )
assert " a c t i v e " in dn1 . succeed ( " s u d o - u h d f s h d f s h a a d m i n - g e t A l l S e r v i c e S t a t e " )
dn1 . succeed ( " s u d o - u h d f s h d f s h a a d m i n - g e t A l l S e r v i c e S t a t e | s y s t e m d - c a t " )
assert " t e s t f i l e c o n t e n t s " in dn1 . succeed ( " s u d o - u h d f s h d f s d f s - c a t / t e s t f i l e " )
2021-10-20 20:33:56 +00:00
2021-11-02 06:29:58 +00:00
nn1 . succeed ( " s y s t e m c t l s t a r t h d f s - n a m e n o d e " )
nn1 . wait_for_open_port ( 9870 )
nn1 . wait_for_open_port ( 8022 )
nn1 . wait_for_open_port ( 8020 )
assert " s t a n d b y " in dn1 . succeed ( " s u d o - u h d f s h d f s h a a d m i n - g e t A l l S e r v i c e S t a t e " )
dn1 . succeed ( " s u d o - u h d f s h d f s h a a d m i n - g e t A l l S e r v i c e S t a t e | s y s t e m d - c a t " )
2021-10-20 20:33:56 +00:00
2021-11-02 06:29:58 +00:00
#### YARN tests ####
2021-10-20 20:33:56 +00:00
2021-11-02 06:29:58 +00:00
rm1 . wait_for_unit ( " n e t w o r k . t a r g e t " )
rm2 . wait_for_unit ( " n e t w o r k . t a r g e t " )
nm1 . wait_for_unit ( " n e t w o r k . t a r g e t " )
2021-10-20 20:33:56 +00:00
2021-11-02 06:29:58 +00:00
rm1 . wait_for_unit ( " y a r n - r e s o u r c e m a n a g e r " )
rm1 . wait_for_open_port ( 8088 )
rm2 . wait_for_unit ( " y a r n - r e s o u r c e m a n a g e r " )
rm2 . wait_for_open_port ( 8088 )
2021-10-20 20:33:56 +00:00
2021-11-02 06:29:58 +00:00
nm1 . wait_for_unit ( " y a r n - n o d e m a n a g e r " )
nm1 . wait_for_open_port ( 8042 )
nm1 . wait_for_open_port ( 8040 )
nm1 . wait_until_succeeds ( " y a r n n o d e - l i s t | g r e p N o d e s : 1 " )
nm1 . succeed ( " s u d o - u y a r n y a r n r m a d m i n - g e t A l l S e r v i c e S t a t e | s y s t e m d - c a t " )
nm1 . succeed ( " s u d o - u y a r n y a r n n o d e - l i s t | s y s t e m d - c a t " )
2021-10-20 20:33:56 +00:00
2021-11-02 06:29:58 +00:00
# Test RM failover
rm1 . succeed ( " s y s t e m c t l s t o p y a r n - r e s o u r c e m a n a g e r " )
assert " s t a n d b y " not in nm1 . succeed ( " s u d o - u y a r n y a r n r m a d m i n - g e t A l l S e r v i c e S t a t e " )
nm1 . succeed ( " s u d o - u y a r n y a r n r m a d m i n - g e t A l l S e r v i c e S t a t e | s y s t e m d - c a t " )
rm1 . succeed ( " s y s t e m c t l s t a r t y a r n - r e s o u r c e m a n a g e r " )
rm1 . wait_for_unit ( " y a r n - r e s o u r c e m a n a g e r " )
rm1 . wait_for_open_port ( 8088 )
assert " s t a n d b y " in nm1 . succeed ( " s u d o - u y a r n y a r n r m a d m i n - g e t A l l S e r v i c e S t a t e " )
nm1 . succeed ( " s u d o - u y a r n y a r n r m a d m i n - g e t A l l S e r v i c e S t a t e | s y s t e m d - c a t " )
2021-10-20 20:33:56 +00:00
2021-11-02 06:29:58 +00:00
assert " E s t i m a t e d v a l u e o f P i i s " in nm1 . succeed ( " H A D O O P _ U S E R _ N A M E = h d f s y a r n j a r $ ( r e a d l i n k $ ( w h i c h y a r n ) | s e d - r ' s ~ b i n / y a r n ~ l i b / h a d o o p - * / s h a r e / h a d o o p / m a p r e d u c e / h a d o o p - m a p r e d u c e - e x a m p l e s - * . j a r ~ g ' ) p i 2 1 0 " )
assert " S U C C E E D E D " in nm1 . succeed ( " y a r n a p p l i c a t i o n - l i s t - a p p S t a t e s F I N I S H E D " )
2021-10-20 20:33:56 +00:00
'' ;
2021-11-02 06:29:58 +00:00
} )
