[C] crypto_dll.c

/*
*       Crypto.dll
*         by Reisyukaku
*
*   Library of encryption/decryption/hashing functions for 3DS roms
*   Last Edited: Nov 23, 2014
*/

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stdint.h>
#include "crypto_dll.h"

enum sha256_opcodes {
  // inputs
  SHA256_IN_BLOCK = 0,
  SHA256_IN_BUFFER = 1,
  SHA256_IN_FILE = 2,
  SHA256_IN_FILENAME = 3,
  // outputs
  SHA256_OUT_WORDS = 0, // 32 bits
  SHA256_OUT_BYTES = 4,
  SHA256_OUT_LONGS = 8, // 64 bits
  SHA256_OUT_STRING = 12,
  // flags
  SHA256_LENGTH_BITS = 16, // length is in bits
  SHA256_REVERSE_WORDS = 32, // reverse word order
  SHA256_UPPERCASE_HEX = 64, // hex digits A-F are uppercase in string outputs
  SHA256_REVERSE_ENDIAN = 64, // when not returning 32-bit words
};

enum sha256_errors {
  SHA256_INVALID_OPCODE = 1,
  SHA256_FILE_NOT_FOUND = 2,
  SHA256_INVALID_ARGUMENT = 3,
  SHA256_IO_ERROR = 4,
  SHA256_PREMATURE_EOF = 5,
};

// SHA-256 says these are "the fractional parts of the square roots of the first eight primes".
// They are, rounded to zero.
#define sha256_init ((uint32_t []) {0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19})

// SHA-256 says these are "the first 32 bits of the fractional parts of the cube roots of the first
// 64 primes". Not putting this one to test. Copy/paste is a wonderful invention.
#define sha256_constants ((uint32_t []) {                                                            \
  0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5,    \
  0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174,    \
  0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da,    \
  0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967,    \
  0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85,    \
  0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070,    \
  0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3,    \
  0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2     \
})

#define sha256_choose_function(one, two, three)    (((one) & (two)) ^ ((~(one)) & (three)))
#define sha256_majority_function(one, two, three)  (((one) & (two)) ^ ((one) & (three)) ^ ((two) & (three)))

#define sha256_circular_shift(number, amount) ((((uint32_t) (number)) >> (amount)) | (((uint32_t) (number)) << (32 - (amount))))

#define sha256_shift_A(number) (sha256_circular_shift(number,  2) ^ sha256_circular_shift(number, 13) ^ sha256_circular_shift(number, 22))
#define sha256_shift_B(number) (sha256_circular_shift(number,  6) ^ sha256_circular_shift(number, 11) ^ sha256_circular_shift(number, 25))
#define sha256_shift_C(number) (sha256_circular_shift(number,  7) ^ sha256_circular_shift(number, 18) ^ ((number) >>  3))
#define sha256_shift_D(number) (sha256_circular_shift(number, 17) ^ sha256_circular_shift(number, 19) ^ ((number) >> 10))

#define sha256_make_word_from_pointer(pointer) (                      \
    (((uint32_t) (0[(const unsigned char *) (pointer)])) << 24) |     \
    (((uint32_t) (1[(const unsigned char *) (pointer)])) << 16) |     \
    (((uint32_t) (2[(const unsigned char *) (pointer)])) <<  8) |     \
     ((uint32_t) (3[(const unsigned char *) (pointer)]))              \
)

//Read Rom and xorpad, xor each byte, and write result to a file.
EXPORT int cryptRomFS(char* path, char* prodCode, char* romname, uint32_t romfsOff, uint32_t fsSize, int mode){
    FILE* fRom, *fXor, *fOut;
    char *xorName = malloc(strlen(path)+strlen(prodCode)+strlen("0.romfs.xorpad")+1);
    char *romName = malloc(strlen(path)+strlen(romname)+1);
    char *outName = malloc(strlen(path)+strlen(prodCode)+strlen("-romfs.bin")+1);
    sprintf(xorName, "%s%s0.romfs.xorpad", path, prodCode);
    sprintf(romName, "%s%s", path, romname);
    sprintf(outName, "%s%s-romfs.bin", path, prodCode);
    fRom = mode % 2 == 0 ? fopen(romName, "rb") : fopen(romName, "wb");
    fOut = mode % 2 == 0 ? fopen(outName, "wb") : fopen(outName, "rb");
    fXor = fopen(xorName, "rb");
    free(xorName);
    free(romName);
    free(outName);
    if (!fRom && !fXor) {
        fclose(fRom);
        fclose(fXor);
        fprintf(stderr, "Error while opening one of the files for reading\n");
        return -1;
    }
    fseek(fRom, romfsOff, SEEK_SET);
    fseek(fXor, 0, SEEK_SET);
    fseek(fOut, 0, SEEK_SET);
    int mUnit = 0x200, //1 3DS media unit
            readBytes = 0,
            i = 0,
            j = 0;
    char romByte[mUnit], xorByte[mUnit], outByte[mUnit];
    printf("wut\n");
    while(!(feof(fXor) || ferror(fXor) || ferror(fRom)) && (i < fsSize)){
        printf("mode = %d | mod = %d\n", mode, mode % 2);
        switch(mode){
            case 0: //decrypt RomFS
                readBytes += fread(romByte, 1, mUnit, fRom);
                readBytes += fread(xorByte, 1, mUnit, fXor);
                for(j = 0; j < mUnit; j++){
                    outByte[j] = romByte[j] ^ xorByte[j];
                }
                fwrite(outByte, 1, mUnit, fOut);
                i+=mUnit;
                break;
            case 1: //encrypt RomFS
                readBytes += fread(xorByte, 1, mUnit, fXor);
                readBytes += fread(outByte, 1, mUnit, fOut);
                for(j = 0; j < mUnit; j++){
                    romByte[j] = outByte[j] ^ xorByte[j];
                }
                fwrite(romByte, 1, mUnit, fRom);
                i+=mUnit;
                break;
            case 2: //dump dec RomFS
                readBytes += fread(romByte, 1, mUnit, fRom);
                readBytes += fread(xorByte, 1, mUnit, fXor);
                for(j = 0; j < mUnit; j++){
                    outByte[j] = romByte[j];
                }
                fwrite(outByte, 1, mUnit, fOut);
                i+=mUnit;
                break;
            case 3: //up dec RomFS
                readBytes += fread(xorByte, 1, mUnit, fXor);
                readBytes += fread(outByte, 1, mUnit, fOut);
                for(j = 0; j < mUnit; j++){
                    romByte[j] = outByte[j];
                }
                fwrite(romByte, 1, mUnit, fRom);
                i+=mUnit;
                break;
        }
    }
    fclose(fRom);
    fclose(fXor);
    fclose(fOut);
    return 0;
}

EXPORT int rehashRomFS(char* path, char* prodCode){
    char *romfsName = malloc(strlen(path)+strlen(prodCode)+strlen("-romfs.bin")+1);
    sprintf(romfsName, "%s%s-romfs.bin", path, prodCode);
    FILE* rfs;
    rfs = fopen(romfsName, "rb+");
    free(romfsName);
    //
    fclose(rfs);
    return 0;
}

EXPORT int sha256 (int op, void * in, void * out, long long skip, long long length) {
  if (op & ~127) return SHA256_INVALID_OPCODE;
  if (!out) return SHA256_INVALID_ARGUMENT;
  if (!((op & SHA256_LENGTH_BITS) || ((op & 3) == SHA256_IN_BLOCK))) length *= 8;
  uint32_t hash[8];
  int status = 0;
  switch (op & 3) {
    case SHA256_IN_FILENAME: {
      if (!in) return SHA256_INVALID_ARGUMENT;
      FILE * file = fopen(in, "rb");
      if (!file) return SHA256_FILE_NOT_FOUND;
      if (skip) {
        status = fseek(file, skip, SEEK_SET);
        if (status) {
          fclose(file);
          return SHA256_IO_ERROR;
        }
      }
      status = sha256_hash_file(file, length, hash);
      fclose(file);
    } break;
    case SHA256_IN_FILE:
      if (!in) return SHA256_INVALID_ARGUMENT;
      if (skip) {
        status = fseek(in, skip, SEEK_CUR);
        if (status) return SHA256_IO_ERROR;
      }
      status = sha256_hash_file(in, length, hash);
      break;
    case SHA256_IN_BUFFER:
      if (length && (!in)) return SHA256_INVALID_ARGUMENT;
      if (length)
        sha256_hash_buffer(((const char *) in) + skip, length, hash);
      else
        sha256_hash_buffer(NULL, 0, hash);
      break;
    case SHA256_IN_BLOCK: {
      void * block;
      void * prevhash;
      if (in) {
        block = ((char *) in) + skip;
        prevhash = ((char *) in) + length;
      } else {
        block = (void *) (intptr_t) skip;
        prevhash = (void *) (intptr_t) length;
      }
      if (length == skip) prevhash = NULL;
      if (!block) return SHA256_INVALID_ARGUMENT;
      memcpy(hash, prevhash ? prevhash : sha256_init, sizeof hash);
      sha256_hash_block(block, hash);
    }
  }
  if (status) return status;
  sha256_format_output(op & ~3, out, hash);
  return 0;
}

static void sha256_format_output (int outformat, void * out, uint32_t * hash) {
  unsigned char p;
  if (outformat & SHA256_REVERSE_WORDS) {
    uint32_t reversed[8];
    for (p = 0; p < 8; p ++) reversed[p] = hash[7 - p];
    memcpy(hash, reversed, sizeof reversed);
  }
  switch (outformat & 12) {
    case SHA256_OUT_STRING:
      sha256_make_result_string(hash, out, outformat & SHA256_UPPERCASE_HEX);
      return;
    case SHA256_OUT_WORDS:
      memcpy(out, hash, 32);
      return;
    case SHA256_OUT_BYTES:
      for (p = 0; p < 7; p ++) sha256_convert_number_to_bytes(hash[p], (char *) out + 4 * p, outformat & SHA256_REVERSE_ENDIAN);
      return;
    case SHA256_OUT_LONGS: {
      uint64_t * result64 = out;
      for (p = 0; p < 4; p ++)
        if (outformat & SHA256_REVERSE_ENDIAN)
          result64[p] = (uint64_t) (hash[2 * p]) | (((uint64_t) (hash[2 * p + 1])) << 32);
        else
          result64[p] = (uint64_t) (hash[2 * p + 1]) | (((uint64_t) (hash[2 * p])) << 32);
    }
  }
}

static void sha256_hash_buffer (const void * buffer, unsigned long long length, uint32_t * result) {
  unsigned long long remaining = length;
  const char * p = buffer;
  memcpy(result, sha256_init, sizeof sha256_init);
  while (remaining >= 512) {
    sha256_hash_block(p, result);
    remaining -= 512;
    p += 64;
  }
  unsigned char block[64];
  if (remaining) memcpy(block, p, (remaining + 7) / 8);
  sha256_pad_block(block, remaining);
  if (remaining >= 448) {
    sha256_hash_block(block, result);
    memset(block, 0, 64);
  }
  unsigned char pos;
  for (pos = 63; length; pos --) {
    block[pos] = length & 0xff;
    length >>= 8;
  }
  sha256_hash_block(block, result);
}

static int sha256_hash_file (FILE * file, unsigned long long length, uint32_t * result) {
  unsigned char block[64];
  memcpy(result, sha256_init, sizeof sha256_init);
  unsigned long long remaining = length;
  int record_length = !length;
  short read_size, read_result = 0;
  while ((remaining || record_length) && !(feof(file) || ferror(file))) {
    if (!record_length && (remaining < 505))
      read_size = (remaining + 7) / 8;
    else
      read_size = 64;
    read_result = fread(block, 1, read_size, file);
    if (ferror(file)) return SHA256_IO_ERROR;
    if ((read_result < read_size) && !record_length) return SHA256_PREMATURE_EOF;
    if (record_length) {
      read_result <<= 3;
      length += read_result;
    } else {
      read_result = (remaining > 512) ? 512 : remaining;
      remaining -= read_result;
    }
    if (read_result < 512) break;
    sha256_hash_block(block, result);
  }
  if (read_result == 512) read_result = 0;
  sha256_pad_block(block, read_result);
  if (read_result >= 448) {
    sha256_hash_block(block, result);
    memset(block, 0, 64);
  }
  for (read_result = 63; length; read_result --) {
    block[read_result] = length & 0xff;
    length >>= 8;
  }
  sha256_hash_block(block, result);
  return 0;
}

static void sha256_pad_block (void * block, unsigned short start_bit) {
  char * p = block;
  p[start_bit / 8] |= 1 << (7 - (start_bit % 8));
  p[start_bit / 8] &= -(1 << (7 - (start_bit % 8)));
  start_bit += 8 - (start_bit % 8);
  if (start_bit < 512) memset(p + start_bit / 8, 0, 64 - start_bit / 8);
}

static void sha256_hash_block (const void * block, uint32_t * partial) {
  uint32_t words[64];
  uint32_t status[8];
  uint32_t x, y;
  unsigned char p;
  // here be dragons
  for (p = 0; p < 16; p ++) {
    words[p] = sha256_make_word_from_pointer(block);
    block = (char *) block + 4;
  }
  for (; p < 64; p ++) words[p] = sha256_shift_D(words[p - 2]) + words[p - 7] + sha256_shift_C(words[p - 15]) + words[p - 16];
  memcpy(status, partial, sizeof status);
  for (p = 0; p < 64; p ++) {
    x = status[7] + sha256_shift_B(status[4]) + sha256_choose_function(status[4], status[5], status[6]) + sha256_constants[p] + words[p];
    y = sha256_shift_A(*status) + sha256_majority_function(*status, status[1], status[2]);
    memmove(status + 1, status, 28);
    status[4] += x;
    *status = x + y;
  }
  for (p = 0; p < 8; p ++) partial[p] += status[p];
}

static void sha256_make_result_string (const uint32_t * hash_result, char * string, int capitalize) {
  string[64] = 0;
  signed char n, b;
  uint32_t current;
  const char * digits = capitalize ? "0123456789ABCDEF" : "0123456789abcdef";
  for (n = 0; n < 8; n ++) {
    current = hash_result[n];
    for (b = 7; b >= 0; b --) {
      string[n * 8 + b] = digits[current & 0xf];
      current >>= 4;
    }
  }
}

static void sha256_convert_number_to_bytes (uint32_t number, char * bytes, int little_endian) {
  unsigned char p;
  for (p = 0; p < 4; p ++) {
    bytes[little_endian ? p : (3 - p)] = number & 0xff;
    number >>= 8;
  }
}