#!/bin/sh ## hyphop ## #= amlogic android image 2 raw image # REQUIRED="device-tree-compiler android-sdk-libsparse-utils libarchive-tools zstd xz-utils" USAGE() { echo "aml2raw convert amlogic android image 2 raw image USAGE: [VARS] aml2raw [DIR|image.[img|7z|xz]] [OUTPUT|-] [--setup|--help|--examples] REQUIRE: dtc simg2img [bsdtar] [zstd] [xz] $REQUIRED KRESCUE: curl krescue/shell/write | sh -s - /tmp/aml2raw/img/image.zst VARS: CT=[zst|xz|raw] - for compression WORK=[path] - work dir download unpack :: DEFAULT ./aml2raw.work TMP2=[path] - temp dd dir :: DEFAULT /tmp/aml2raw/$PID USR=[path] - usb bin dir :: DEFAULT ~/aml2raw ACFG=[filename] - android image config :: DEFAULT image.cfg REWRITE= - enable output file rewrite :: DEFAULT no NOCLEANUP= - no cleanup flag for TMP2 :: DEFAULT no WORKCLEAN= - clean up working dir after done :: DEFAULT no MKDIR=[no|yes] - make unexist WORK dir :: DEFAULT yes SIMG= - force use custom simg2img FLAG :: DEFAULT no NOSINGLE= - no single stream archive FLAG :: DEFAULT no NOTE: fully supported only from Krescue 201219_126 Sat 19 Dec 2020 03:10:58 AM UTC EXTRA VARS for write meta BUILDER - LABEL - DESC - LINK - " } EXAMPLES() { cat <<"end" EXAMPLES: ## ANDROID developers # download install script cd ~ wget https://raw.githubusercontent.com/hyphop/khadas-rescue-tools/master/tools/aml2raw chmod 0777 aml2raw # just check build process from burn dir project aml2raw VIM3L_Pie_V201113.amlimage.unpackdir # .raw.img.xz build from burn dir project - VIM3L_Pie_V201113.amlimage.unpackdir ./aml2raw VIM3L_Pie_V201113.amlimage.unpackdir VIM3L_Andorid.Pie_V201113.raw.img.xz # .raw.img.xz build from burn dir project + custom meta + and custom config name LABEL="Android" \ DESC="Android Pie last version from 201113 ..." \ BUILDER="Terry" \ LINK="khadas.com" \ ACFG="aml_upgrade_package_avb.conf" ./aml2raw android_image_dir VIM3L_Andorid.Pie_V201113.raw.img.xz # raw image output CT=raw aml2raw ../img/Android_Pie.VIM1/dl/VIM1_Pie_V200317.7z # convert to raw image # raw image output to stdout CT=raw aml2raw ../img/Android_Pie.VIM2/dl/VIM2_Pie_V200624.7z - # output to stdout # zst image output to stdout + compression detected from output extension name aml2raw VIM3_Pie_V201112.7z - > VIM3_Pie_V201112.zst # xz image output to file + compression detected from output extension name aml2raw VIM3_Pie_V201112.7z VIM3-Android-Pie_V201112.raw.img.xz # online convert example aml2raw https://dl.khadas.com/Firmware/VIM3/Android/VIM3_Pie_V201112.7z VIM3-Android-Pie_V201112.raw.img.xz CT=xz aml2raw http://dl.khadas.com/products/vim4/firmware/android/vim4-android-11-64bit-v220826.img.xz aml2raw http://dl.khadas.com/.test/vim1s-android-11-v220901.img.xz vim1s-android-11-v220901.raw.img.xz # raw image output to file + compression detected from output name extension (img) aml2raw VIM3_Pie_V201112.7z VIM3_Pie_V201112.img # ATV VIM3L image from SC convert example LABEL=Android-TV BUILDER=SC \ aml2raw SC_VIM3L_USER_ATV-v3.0-20201125.7z VIM3L-Android-TV-SC_USER-v3.0-20201125.raw.img.xz # ATV VIM3 image from SC convert example LABEL=Android-TV BUILDER=SC \ aml2raw SC_VIM3_USER_ATV-v3.0-20200803.7z VIM3-Android-TV-SC_USER_ATV-v3.0-20200803.raw.img.xz # ATV VIM2 aimge from SC convert example end } CLEAN(){ MSG "CLEANUP WORK DIR $WORK" ## carifull clean [ "$PACKED" ] && { RM "$PACKED" "$PACKED.md5sum" } [ "$UNPACKED" ] && \ RM "$UNPACKED" RM "$WORK"/*.unpacked RM "$WORK"/*.amlimage RM "$WORK"/*.unpackdir/* RM "$WORK"/*.unpackdir } [ "$ACFG" ] || \ ACFG=image.cfg # BOOT RES BOOT_START=0 RES_START=0x02400000 DTB_START=0x02800000 DT2_START=0x02840000 # ENV_START=0x4d400000 # LOGO_START=0x4e400000 #CACHE_START=0x06c00000 UND_START=0x06c00000 UND_SIZE=0x00800000 #PART_OFFSET=0x4e400000 PART_OFFSET=0 BE=0x80000 BC=0x800000 BS=0x400000 CMD(){ echo "# $@" >&2 $@ } MSG(){ echo "$@">&2 } DIE(){ echo "[e] DIE $@" >&2 exit 1 } RM(){ # safe clean echo "[RM] $# : $@" >&2 [ "$#" = 0 ] && exit for r in "$@"; do [ -e "$r" ] || continue [ ${#r} -lt 3 ] && DIE "unsafe del" [ "${r%/}/" = "$r" ] && DIE "unsafe dir del" MSG "[i] remove $r" if [ -d "$r" ] ; then rmdir "$r" else rm "$r" fi done } chk_a(){ [ "$bsdtar" ] || \ bsdtar="$(which bsdtar)" [ $? = 0 ] || DIE "bsdtar not found ! sudo apt-get install libarchive-tools" } chk_p(){ b=aml_image_v2_packer [ "$aml_image_v2_packer" ] || \ aml_image_v2_packer=$(which $b) [ $? = 0 ] && return o="$USR/bin/$b" c="$o.gz" MSG "[i] download $b => $c" curl -jkL -C- -o "$c" \ https://github.com/hyphop/khadas-rescue-tools/releases/download/dl/$b.gz [ -e "$o" ] || { gzip -d $c || DIE "unpack $c" chmod 0777 "$o" } } chk_s(){ b=simg2img o="$USR/bin/$b" # force use custom simg2img [ "$SIMG" ] && simg2img="$o" [ "$simg2img" ] || { simg2img=$(which $b) [ $? = 0 ] && return } [ -x "$o" ] && return c="$o.gz" MSG "[i] download $b => $c" curl -jkL -C- -o "$c" \ https://github.com/hyphop/khadas-rescue-tools/releases/download/dl/$b.gz [ -e "$o" ] || { #sudo apt-get install android-sdk-libsparse-utils gzip -dc $c > $o || DIE "unpack $c" chmod 0777 "$o" } } chk_z(){ b=xze o="$USR/bin/$b" [ "$xze" ] || \ xze=$(which $b) [ $? = 0 ] && return MSG "[i] download $b" curl -jkL -C- -o "$o" \ https://raw.githubusercontent.com/krescue/krescue/master/tools/xze [ -x "$o" ] || \ chmod 0777 "$o" } chk_xz(){ b=xz o="$USR/bin/$b" [ "$xz" ] || \ xz=$(which $b) #[ $? = 0 ] && return CHK_="5.3.1" $xz -h 2>&1 | grep -q threads || xz= #$xz -V 2>&1 | grep -q $CHK_ || xz= [ -e "$o" ] && { [ -x "$o" ] || \ chmod 0777 "$o" } [ -x "$o" ] && return c="$o.gz" MSG "[i] download $b => $c" curl -jkL -C- -o "$c" \ https://github.com/hyphop/khadas-rescue-tools/releases/download/dl/$b.gz [ -e "$o" ] || { gzip -dc $c > $o || DIE "unpack $c" chmod 0777 "$o" } } DTC_VER= chk_dtc(){ b=dtc o="$USR/bin/$b" [ "$dtc" ] || \ dtc=$(which $b) #[ $? = 0 ] && return CHK_="1.5.0" DTC_VER=$($dtc -v 2>&1) echo $DTC_VER | grep -q $CHK_ || dtc= [ -e "$o" ] && { [ -x "$o" ] || chmod 0777 "$o" [ -x "$o" ] && return } c="$o.gz" MSG "[i] download $b => $c" curl -jkL -C- -o "$c" \ https://github.com/hyphop/khadas-rescue-tools/releases/download/dl/$b.gz [ -e "$o" ] || { gzip -dc $c > $o || DIE "unpack $c" chmod 0777 "$o" } } SETUP(){ CMD="sudo apt-get install $REQUIRED" MSG "[i] setup need root privs for next command" MSG "# $CMD" $CMD } case "$@" in *--help*|-h) USAGE && exit ;; *--setup*) SETUP exit ;; *--exampl*) EXAMPLES exit ;; esac # default yes [ "$MKDIR" ] || MKDIR=yes # replace no to EMPTY [ "$MKDIR" = "no" ] && MKDIR= [ "$NOCLEANUP" = "no" ] && NOCLEANUP= [ "$WORKCLEAN" = "no" ] && WORKCLEAN= [ "$SIMG" = "no" ] && SIMG= gen_name(){ case $1 in # *EMMC*) *) INF=$(basename "$1") for s in amlimage dir 7z img xz; do INF=${INF%%.$s} done INF=${INF#VIM1?} INF=${INF#VIM2?} INF=${INF#VIM3L?} INF=${INF#VIM3?} INF=${INF#VIM?} MSG "[i] INF: $INF" ;; esac } [ "$WORK" ] || \ WORK=$(realpath ~/${0##*/}.work) [ "$USR" ] || \ USR=$(realpath ~/${0##*/}) mkdir -p $USR MSG "[i] aml2raw image converter starting ..." MSG "[i] USR : $USR" [ "$MKDIR" ] && { mkdir -p $WORK || DIE "cant create working dir $WORK" } [ "$WORK" ] && { WORK=$(realpath "$WORK") } [ "$WORKCLEAN" = "EXIT" ] && { CLEAN exit } [ "$TMP2" ] || \ TMP2=/tmp/${0##*/} mkdir -p $TMP2 MSG "[i] TMP2 : $TMP2" [ "$TMP2" ] && { for t in $TMP2/*; do [ -d "$t" ] || continue p=${t##*/} [ -d "/proc/$p" ] || { MSG "[i] clean old unused cache dir $t" rm -rf "$t" } done } #exit 0 case $1 in http*) Z="$WORK/$(basename "$1")" #gen_name "$Z" [ -s $Z.md5sum ] || { MSG "[i] download $1 > $Z" curl -jkL -C- -o "$Z" "$1" || DIE "download $1" md5sum "$Z" > $Z.md5sum } IN=$Z # shift ;; esac [ "$IN" ] || \ IN=$1 [ "$IN" ] || { USAGE && DIE "input not defined" } [ -e "$IN" ] || DIE "not found $IN" [ "$2" ] && CO="$(realpath "$2")" mkdir -p "$USR/bin" export PATH="$USR/bin:$PATH" MSG "[i] WORK: $WORK " MSG "[i] OUT: $CO " O="$WORK"/update.img INF=${IN%.*} PACKED= UNPACKED= while [ ! -d "$IN" ]; do case $(file -L "$IN") in *7-zip*) O=${IN%.*}.amlimage O="$WORK/${O##*/}" PACKED="$IN" UNPACKED="$O" UN="$O".unpacked [ -e "$UN" ] || { MSG "[i] 7z unpack $IN to $O" chk_a $bsdtar -O -xf "$IN" '*.img' | dd conv=sparse status=progress of="$O" || DIE "unpack $IN" date > "$UN" } IN=$O ;; *XZ*) O=${IN%.*}.amlimage O="$WORK/${O##*/}" PACKED="$IN" UNPACKED="$O" UN="$O".unpacked [ -e "$UN" ] || { MSG "[i] xz unpack $IN to $O" chk_xz xz -dc "$IN" > "$O" date > "$UN" } IN=$O ;; *) D="$WORK/${IN##*/}".unpackdir UN="$D".unpacked [ -d "$D" ] || rm $UN mkdir -p "$D" chk_p chk_p MSG "CHK UN $UN" [ -s "$UN" ] || { MSG "[i] aml unpack $IN => $D" head -c4096 "$IN" | grep -q PARTITION || DIE "unknown input $IN" $aml_image_v2_packer -d "$IN" "$D" 1>&2 || DIE "aml unpack $IN" date > "$UN" } cd $D [ "$UNPACK_ONLY" ] && exit 0 break esac done MSG "[i] IN: $IN" [ -d "$IN" ] && cd $IN [ -s $ACFG ] || \ DIE "config $ACFG not found" # linux detect tail -1 $ACFG | grep rootfs && LABEL=${LABEL:-Linux} grep "sub_type=\"recovery\"" $ACFG && LABEL=${LABEL:-Android} chk_s chk_s #$simg2img ## BEGIN while read l ; do case $l in *main_type=\"UBOOT\"*) f=${l#*\"} f=${f%%\"*} UBOOT=$f MSG "[i] UBOOT : $f" ;; *sub_type=\"bootloader\"*) f=${l#*\"} f=${f%%\"*} UBOOT=$f dd if=$UBOOT count=3 bs=1 2>/dev/null | grep "@ML" && UTEST="-DDR-" case "$UBOOT$UTEST" in *DDR*|*USB*) CMD dd if=$UBOOT seek=1 of=$UBOOT.fix UBOOT=$UBOOT.fix echo "[i] fix uboot $UBOOT + $UTEST" ;; esac MSG "[i] UBOOT : $f" ;; *main_type=\"dtb\"*) f=${l#*\"} f=${f%%\"*} DTB=$f MSG "[i] DTB : $f" ;; esac done < $ACFG #cat /sys/class/block/mmcblk2/device/erase_size # 524288 # 0x80000 #cat /sys/class/block/mmcblk2/device/preferred_erase_size # 4194304 #0x400000 [ "$DTB" ] || \ DTB=_aml_dtb.PARTITION #[ "$DTS" ] || \ # DTS=_aml_dtb.PARTITION.dts chk_dtc chk_dtc [ "$dtc" ] || DIE "dtc $DTC_VER" $dtc -v || DIE "dtc not valid" [ "$DTB" ] && { file $DTB | grep gzip && { gzip --quiet -dc $DTB > $DTB.raw DTB2=$DTB DTB=$DTB.raw } INPUT="$dtc -Idtb -q $DTB" $INPUT 1>/dev/null || { echo "# try multy dtb AML" dd if=$DTB skip=$((0x00000800/512)) of=$DTB.raw DTB=$DTB.raw INPUT="$dtc -Idtb -q $DTB" $INPUT 1>/dev/null || DIE "dtb broken $DTB" } } #### #INPUT=" #BOARD_=$($dtc -q $DTB | grep -m1 -i amlogic-dt-id ) BOARD_=$($dtc -Idtb -q $DTB | grep -m1 -i "compatible" ) MSG "[d] board $BOARD_" ptail="tail -n+2" case $BOARD_ in *t7*) BOARD=VIM4 ptail="cat" ;; *vim4*) BOARD=VIM4 ptail="cat" ;; *vim3l*|*VIM3L*) BOARD=VIM3L ;; *vim2*|*VIM2*) BOARD=VIM2 ;; *vim3*|*VIM3*|*g12b*) BOARD=VIM3 ;; *vim1s*|*VIM1S*) BOARD=VIM1S ;; *vim*|*VIM*) BOARD=${BOARD-VIM1} ;; *) DIE "unknown board $BOARD_" exit 1 esac #" MSG "[i] BOARD : $BOARD" MSG "[i] TMP2 : $TMP2" [ "$DTS" ] && \ INPUT="cat $DTS" P="$TMP2"/$$/parts.dts.info T="$TMP2"/$$/img mkdir -p $T MSG "[i] $P" # extract parition info !!! NOTE not super solution $INPUT | \ grep -m1 partitions -A300 | \ grep -B300 -A2 0xffffffff | \ grep -e pname -e mask -e size -e phandle | $ptail > $P # sort phandles grep phandle $P | sort > $P.parts # reorder parts while read l ; do grep -B3 "$l" $P done < $P.parts > $P.sort # done mv $P.sort $P cat $P #exit 1 BS=512 DD() { CMD dd status=none conv=fsync,notrunc if="$1" of="$2" $3 $4 $5 $6 } DZ() { [ "$3" ] || DIE DZ $@ DD /dev/zero "$2" seek=$(($3/BS-1)) count=1 case "$1" in /dev/null|/dev/zero|"") return ;; esac DD "$1" "$2" } DO() { DD "$1" "$2" seek=$(($3/BS)) } chk_xz chk_xz CMP_XZ="$xz -v -T4 --block-size 90M -k -c -F xz -7" MSG "[XZ] $xz" CMP() { case $CT in zst) CMP_="zstd -T0 -5 -c" MSG "[i] $CMP_" $CMP_ "$1" >> "$2" ;; xz) CMP_=$CMP_XZ MSG "[i] $CMP_" $CMP_ "$1" >> "$2" ;; raw) [ "$2" = "-" ] && { MSG "[i] RAW stdout $1" cat "$1" return } MSG "[i] RAW copy $1 > $2" cat "$1" >> "$2" ;; *) DIE undefined compress method esac } [ "$CT" ] || \ CT=zst RO=$T/image [ "$CO" ] || { CO=$RO.$CT } PID=$$ [ "$CO" = "-" ] && { co=/proc/$PID/fd/1 STDOUT=$(readlink $co) MSG "[i] output to STDOUT $STDOUT" case $STDOUT in */dev/pts*) DIE "bin output to console REJECTED" ;; esac CO=$co } rm -f $RO MSG "[i] image output to $CO" for cc in $CO $STDOUT; do case $cc in *.xz|*.zst) CT=${cc##*.} MSG "[i] force output $CT compressed" ;; *.img) CT=raw MSG "[i] force output $CT" ;; esac done MSG "[i] LABEL: $LABEL" case $CO in */.raw*) PRE=${CO%/*} POST=${CO##*/.raw} MSG "INF: $INF" gen_name "$IN" MSG "INF: $INF" CO=$PRE/$BOARD-$LABEL-$INF.raw$POST MSG "[i] auto rename output to $CO" ;; esac #exit 0 #cp $UBOOT $T/image #CMD truncate -s $((LOGO_START)) $T/image #CMP_SINGLE=1 [ "$NOOP" ] && exit [ -f $CO ] && { [ "$REWRITE" ] || DIE "output file $CO exist need enable REWRITE flag" rm -f $CO 2>/dev/null } BOOT_WRITED= write_boot(){ MSG "write boot end is $(printf %06x $1) = $1" END=$1 DZ $UBOOT $RO $END if [ "$DTB2" ]; then DO $DTB2 $RO $DTB_START DO $DTB2 $RO $DT2_START else DO $DTB $RO $DTB_START DO $DTB $RO $DT2_START fi CMP $RO $CO BOOT_WRITED=1 } CACHE_SIZE=0 CONVERT(){ MSG "1st scan" # first scan while read l ; do case $l in *pname*) a=${l#*\"} pname=${a%\"*} ;; *size*) a=${l##* } size=${a%\>*} case "$pname" in cache) CACHE_SIZE=$size ;; esac esac done < $P MSG "2nd scan" pname= size= # second scan while read l ; do case $l in *pname*) a=${l#*\"} pname=${a%\"*} #[ "$pname" = "logo" ] && exit 1 ;; *size*) a=${l##* } size=${a%\>*} [ "$PART_OFFSET" = 0 ] && { case $LABEL in *nux*) PART_OFFSET=$((UND_START+size+BC)) case $BOARD in VIM4|VIM1S) PART_OFFSET=0x08400000 ;; esac write_boot $PART_OFFSET ;; *) case $BOARD in VIM4|VIM1S) PART_OFFSET=0x08400000 ;; *) MSG "CACHE SIZE $CACHE_SIZE" PART_OFFSET=$((UND_START+$CACHE_SIZE+size+BC+BC)) ;; esac write_boot $PART_OFFSET ;; esac } b=$PART_OFFSET o=$((PART_OFFSET+size)) P=$pname.PARTITION case "$pname" in cache) continue ;; data|userdata) # clean data 64M size=$((1024*1024*64)) o=$((PART_OFFSET+size)) #break ;; esac printf "[i] %10s %10x(%10d) %10x(%10d) %10x\n" $pname $b $((b/512)) $((size/512)) $size $o >&2 PD=/dev/zero PART_TYPE=_BLANKED_ [ -s "$P" ] || { # get partition file from config P_=$(grep "sub_type=\"$pname\"" $ACFG | grep -m1 -v \^\#) P_=${P_#*\"} P_=${P_%%\"*} [ -e "$P_" ] && { P="$P_" } } [ -s "$P" ] && \ PART_TYPE=$(file -L "$P") MSG "> $PART_TYPE" SPARSE= [ -s "$P" ] && { case $PART_TYPE in *sparse*) #PD=$P.raw PD=$TMP2/$P.raw CMD $simg2img "$P" "$PD" SPARSE=1 [ -L "$P" ] && \ MSG "[i] $P -> $(readlink "$P")" ls -L -l1 "$P" "$PD" >&2 ;; # *buildvariant=*) # LABEL=Android-TV # BUILDER=SC # MSG "[i] BOARD $BOARD => $LABEL+$BUILDER || PTYPE: $PTYPE // $INF" # PD="$P" # ;; *) PD="$P" ;; esac } PART_OFFSET=$((o+BC)) #continue rm $RO ############# part size padded DZ "$PD" $RO $((PART_OFFSET-b)) # [ ! "$NOCLEAN" ] && { # [ -f "$PD" ] && CMD rm -f "$PD" "${PD%.raw}" # } [ "$SPARSE" ] && \ [ -e "$PD" ] && \ rm "$PD" MSG "=$pname" CMP $RO $CO [ -s $P ] && ls -L -l1 $P >&2 [ "$pname" = "data" ] && break esac done < $P } case $CT in xz) [ "$NOSINGLE" ] && { MSG "[w] XZ multy stream output!!!" CONVERT } [ "$NOSINGLE" ] || { # single stream compress MSG "[i] XZ single stream output" CT=raw CO=- CONVERT | $CMP_XZ > $CO } ;; *) CONVERT esac case $CO in -) MSG "[i] PIPE output - end" ;; /proc/*) MSG "[i] PIPE output $STDOUT end" #cat /proc/$PID/fdinfo/1 >&2 ;; *) ls -l1 $CO >&2 esac [ "$LABEL" ] || \ LABEL=android [ "$BOARD" ] || \ BOARD=VIMx [ "$BUILDER" ] || \ BUILDER=Khadas [ "$LINK" ] || \ LINK=http://dl.khadas.com [ "$CMP_PID" ] && { MSG "[i] wait $CMP_PID" wait $CMP_PID } [ "$DESC" ] || \ DESC="$LABEL ${INF##*/} for Khadas $BOARD board" MSG "[i] BOARD $BOARD => $LABEL+$BUILDER INF: ${INF##*/} // LINK: $LINK DESC: $DESC ========================================================= " ## add meta case $CO in *.xz) chk_z chk_z xz=$xz $xze $CO -9 --meta \ BOARD="$BOARD" \ LABEL="$LABEL" \ BUILDER="$BUILDER" \ LINK="$LINK" \ DESC="$DESC" \ || DIE "xze fail" ;; *) ;; esac [ "$NOCLEANUP" ] || rm -rf $TMP2 [ "$WORKCLEAN" ] || exit CLEAN exit exit 0 <