yarn_mem_track.sh

#!/bin/bash

YARN_ENV=$HADOOP_HOME/etc/hadoop/yarn-env.sh
CONTAINER_PATTERN="container_*"
MEM_GREP_PATTERN="INFO org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorImpl: Memory usage of ProcessTree"

# log example
# yarn-wdong-nodemanager-klose1.log:2015-01-05 04:00:21,921 INFO org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorImpl: Memory usage of ProcessTree 27966 for container-id container_1420401494322_0001_01_000001: 78.1 MB of 2 GB physical memory used; 1.6 GB of 4.2 GB virtual memory used

# we want to use awk convert this to
# klose1 2015-01-05 04:00:21 TS 27966 container_1420401494322_0001_01_000001 78.1 MB 2 GB 1.6 GB 4.2 GB
# where TS is equivalent timestamp
# $1: yarn-wdong-nodemanager-klose1.log:2015-01-05
# $2: 04:00:21,921
# $9: 27966
# $12: container_1420401494322_0001_01_000001
# $13: 78.1
# $14: MB
# $16: 2
# $17: GB
# $21: 1.6
# $22: GB
# $24: 4.2
# $25: GB

# output format is
# washtenaw 2015-01-06 14:56:40 1420574200 16669 container_1420574192658_0001_01_000001 0.274023 9 8.9 18.9

# extract node from $1:

AWK_PROG='
    function calc_mem (n, unit) {
        if (unit == "B") {
            return n / 1024.0 / 1024.0 / 1024.0;
        }
        if (unit == "KB") {
            return n / 1024.0 / 1024.0;
        }
        if (unit == "MB") {
            return n / 1024.0;
        }
        if (unit == "GB") {
            return n;
        }
        print "Found memory unit of neither MB or GB, do not know what to do." > "/dev/stderr";
        print $0 > "/dev/stderr";
    }
    {
    split($1, arr, "[:]");
    node = arr[1];
    date = arr[2];
    split(node, arr, "[-.]");
    node=arr[4];
    split($2, arr, ",");
    time = arr[1];
    date_time=date  " "  time;
    gsub(/[:-]/, " ", date_time);
    ts = mktime(date_time);
    pid=$9;
    split($12, arr, ":");
    container = arr[1];
    p_use = calc_mem($13, $14);
    p_cap = calc_mem($16, $17);
    v_use = calc_mem($21, $22);
    v_cap = calc_mem($24, $25);
    print node, date, time, ts, pid, container, p_use, p_cap, v_use, v_cap;
} '

if [ -z "$1" ]
then
    echo "usage:    $0 app-id app-id ..."
    exit 1
fi

if [ ! -f $YARN_ENV ]
then
    echo "yarn-env.sh not found under \$HADOOP_HOME/etc/hadoop/yarn-env.sh"
    echo "\$HADOOP_HOME=$HADOOP_HOME"
    exit 1
fi

if [ -z "$HADOOP_YARN_HOME" ]
then
    HADOOP_YARN_HOME=$HADOOP_HOME
fi

. $YARN_ENV

LOG_DIR=$YARN_LOG_DIR/userlogs/$APP

if [ ! -d $LOG_DIR ]
then
    echo "Log for app $APP not found under $LOG_DIR"
    exit 1
fi

NC=`find $LOG_DIR/ -type d -name "$CONTAINER_PATTERN" | wc -l`

MEMLOG=`mktemp`

grep "$MEM_GREP_PATTERN"  $YARN_LOG_DIR/yarn-*.log | sed 's/0B of/0 GB of/g' | awk "$AWK_PROG" > $MEMLOG

SUM=`mktemp`

while true
do

APP=$1

shift

if [ -z "$APP" ]; then break; fi

> $SUM

echo $NC containers found for app $APP
NC=1
find $LOG_DIR/ -type d -name "$CONTAINER_PATTERN" | sort | while read C
do
    CID=`basename $C`
    printf "%s: " $CID
    NC=$((NC+1))
    grep $CID $MEMLOG | cut -f 7,8 -d ' ' | awk 'BEGIN{m=0;c=0;}{if ($1 > m) {m = $1;} c=$2;}END{print $1, $2}' | while read U C
    do
        echo $U of $C GB
        echo $U $C >> $SUM
    done
done

awk 'BEGIN{u=0;c=0;}{u+=$1;c+=$2;}END{print u, c;}' $SUM | while read U C
do
    echo Total: $U of $C GB.
done

echo
done

rm $MEMLOG