Tuesday 10 September 2019

SCHEMA_GENERATION_WRAPPER_SRIPT


#Source DBSRVR config file
#Taking user input and passing to DBSRVR file

echo $0
BIN_DIR=`dirname $0`
echo $BIN_DIR
source $BIN_DIR/cdl-bl-db-env.conf $2
#Source Hadoop Conf file
source $BIN_DIR/cdl-bl-env.conf

#Source the parameter file
echo $1
source $1

#Job Unix Log File Location
JobRunDate=`date +%Y:%m:%d:%H:%M:%S`
SummaryLogFileNm=SCHEMA_GENERATION_Wrapper_`date +%Y%m%d%H%M%S`_${TBL_NM}_$USER.log
SLogFullFileName=$CDL_LOGS/${SummaryLogFileNm}
exec 1> $SLogFullFileName 2>&1


echo  $CDL_CONF
echo  $CDL_PARAM
echo  $CDL_LOGS


#Source bash_profile to get credentials for Teradata BTEQ Script
source $HOME/.bash_profile

echo extracting columns started
sh $CDL_BIN/SCHEMA_DIR/EXTRACT_COLUMNS.sh $1 $2
if [ $? -eq 0 ]; then
echo extracting columns completed
else
echo extracting columns failed
exit 1
fi


echo avro schema genertion started
sh $CDL_BIN/SCHEMA_DIR/AVRO_SCHEMA_GENERATION.sh $1 $2
if [ $? -eq 0 ]; then
echo avro schema genertion completed
else
echo avro schema genertion failed
exit 1
fi


echo Parquet schema genertion started
sh $CDL_BIN/SCHEMA_DIR/PARQUET_SCHEMA_GENERATION.sh $1 $2
if [ $? -eq 0 ]; then
echo Parquet schema genertion completed
else
echo Parquet schema genertion failed
exit 1
fi

if [ "${ENVIRONMENT}" == "pr" ]
then
allphi_schema=${ENVIRONMENT}_${PROGRAM}${PROJECT_ABBREVATION}${PHI_NOPHI_DB}_r000_wh_allphi
else
allphi_schema=${ENVIRONMENT}_${PROGRAM}${PROJECT_ABBREVATION}${PHI_NOPHI_DB}_${HIVE_GBD_NOGBD}_r000_wh_allphi
fi

if [ "${PROJECT_ABBREVATION_AR}" == "pdlr_ar" ]
then
cdl_schema_name=${ENVIRONMENT}_${PROGRAM}${PROJECT_ABBREVATION}${PHI_NOPHI_DB}_${HIVE_GBD_NOGBD}_r000_ar
else
cdl_schema_name=${cdl_schema_name_wh}
fi
echo "allphi_schema: ${allphi_schema}"
echo "cdl_schema_name: ${cdl_schema_name}"

#Create view
beeline -u $BEELINE_CMD  --outputformat=tsv2 --showHeader=false hive -e "CREATE OR REPLACE VIEW ${allphi_schema}.${cdl_table_name} AS select * from ${cdl_schema_name}.${cdl_table_name};"
#Check Script successfully executed or not
if [ $? -eq 0 ]; then
    echo "Succefully create the view ${allphi_schema} "
else

echo "Beeline command  failed create the existing view  ${allphi_schema}"
#exit 1
fi

1 comment:

spark_streaming_examples

Create Spark Streaming Context: ========================================== scala: --------------- import org.apache.spark._ import ...