1、下载

PE文件，控制台程序

2、main函数

大致分析，请看下面的注释

int __cdecl main(int argc, const char **argv, const char **envp)
{
  unsigned int str_length; // eax
  unsigned int myflag_length; // eax
  void *v5; // rax
  void *v7; // rax
  int i; // [rsp+24h] [rbp-D4h]
  void *memory; // [rsp+28h] [rbp-D0h]
  char myflag[32]; // [rsp+30h] [rbp-C8h] BYREF
  char Str[128]; // [rsp+50h] [rbp-A8h] BYREF

  strcpy(Str, "12345678abcdefghijklmnopqrspxyz");
  memset(&Str[32], 0, 0x60ui64);
  memset(myflag, 0, 0x17ui64);
  sub_1400054D0("%s", myflag);                  // 输入字符串
  memory = malloc(0x408ui64);                   // v9应该是一个内存地址，是申请下来的地址
  str_length = strlen(Str);
  sub_140001120(memory, Str, str_length);       // 关键
  myflag_length = strlen(myflag);
  sub_140001240(memory, myflag, myflag_length); // 关键
  for ( i = 0; i < 22; ++i )
  {
    if ( ((unsigned __int8)myflag[i] ^ 0x22) != main_break[i] )// 加密之后的myflag[i]^0x22=main_break[i]
    {
      v5 = (void *)sub_1400015A0(&off_14013B020, "error");// 输出错误
      _CallMemberFunction0(v5, sub_140001F10);
      return 0;
    }
  }
  v7 = (void *)sub_1400015A0(&off_14013B020, "nice job");// 正确
  _CallMemberFunction0(v7, sub_140001F10);
  return 0;
}

sub_140001120函数和sub_140001240函数看起来很重要

提取main_break[]数组

unsigned char main_break[] =
{
  0x9E, 0xE7, 0x30, 0x5F, 0xA7, 0x01, 0xA6, 0x53, 0x59, 0x1B, 
  0x0A, 0x20, 0xF1, 0x73, 0xD1, 0x0E, 0xAB, 0x09, 0x84, 0x0E, 
  0x8D, 0x2B, 0x00, 0x00
};

3、sub_140001120函数

__int64 __fastcall sub_140001120(_DWORD *a1, __int64 key, int key_length)
{
  __int64 result; // rax
  int i; // [rsp+0h] [rbp-28h]
  int j; // [rsp+0h] [rbp-28h]
  int v6; // [rsp+4h] [rbp-24h]
  int v7; // [rsp+8h] [rbp-20h]
  int tmp; // [rsp+Ch] [rbp-1Ch]
  _DWORD *s_box; // [rsp+10h] [rbp-18h]

  *a1 = 0;
  a1[1] = 0;
  s_box = a1 + 2;    			//给s_box放在堆空间里面
  for ( i = 0; i < 256; ++i )
    s_box[i] = i;                               // 给 v9数组初始化 0~255
  v6 = 0;
  result = 0i64;
  LOBYTE(v7) = 0;
  for ( j = 0; j < 256; ++j )
  {                                         // 下面这一段是RC4加密，v8是临时变量，进行交换
    tmp = s_box[j];
    v7 = (key[v6] + tmp + v7);            // Key[v6]+tmp+v7
    s_box[j] = s_box[v7];
    s_box[v7] = tmp;
    if ( ++v6 >= key_length )                   // v6下标在规定范围内255循环
      v6 = 0;                                   // 重置为0
    result = (unsigned int)(j + 1);
  }
  return result;                                // 返回一个result
}

可以得出结论，sub_140001120是re4_init函数

Str="12345678abcdefghijklmnopqrspxyz" 是key

4、sub_140001240函数

那么这个函数就是RC4加密函数了，稍微分析了一下，发现写的代码很妙啊

_DWORD *__fastcall sub_140001240(_DWORD *memory, __int64 myflag, int myflag_length)
{
  _DWORD *result; // rax
  int i; // [rsp+0h] [rbp-28h]
  int tmp1; // [rsp+4h] [rbp-24h]
  int tmp2; // [rsp+8h] [rbp-20h]
  int v7; // [rsp+Ch] [rbp-1Ch]
  int v8; // [rsp+10h] [rbp-18h]
  _DWORD *s_box; // [rsp+18h] [rbp-10h]

  tmp1 = *memory;
  tmp2 = memory[1];
  s_box = memory + 2;
  for ( i = 0; i < myflag_length; ++i )
  {                                             // 这段RC4写的很妙啊
    tmp1 = (unsigned __int8)(tmp1 + 1);
    v7 = s_box[tmp1];
    tmp2 = (unsigned __int8)(v7 + tmp2);
    v8 = s_box[tmp2];
    s_box[tmp1] = v8;                           // 交换s_box盒
    s_box[tmp2] = v7;
    *(_BYTE *)(myflag + i) ^= LOBYTE(s_box[(unsigned __int8)(v8 + v7)]);// myflag[i] ^=s_box[v8+v7]
  }
  *memory = tmp1;
  result = memory;
  memory[1] = tmp2;
  return result;
}

5、解题

网上找一下RC4的解题脚本，做RC4这类题，只要找到密文和 key，就可以做出来

flag1=[]
main_break=[0x9E, 0xE7, 0x30, 0x5F, 0xA7, 0x01, 0xA6, 0x53, 0x59, 0x1B,0x0A, 0x20, 0xF1, 0x73, 0xD1, 0x0E, 0xAB, 0x09, 0x84, 0x0E,0x8D, 0x2B, 0x00, 0x00]
for i in range(len(main_break)):
    flag1.append(main_break[i]^0x22)
#flag1是经过rc4加密后的值
#现在有 flag1 和 rc4 key
#输出
flag1= [188, 197, 18, 125, 133, 35, 132, 113, 123, 57, 40, 2, 211, 81, 243, 44, 137, 43, 166, 44, 175, 9, 34, 34]

#include<stdio.h>
#include<string.h>
typedef unsigned longULONG;

/*初始化函数*/
void rc4_init(unsigned char*s, unsigned char*key, unsigned long Len)
{
	int i = 0, j = 0;
	char k[256] = { 0 };
	unsigned char tmp = 0;
	for (i = 0; i < 256; i++)
	{
		s[i] = i;
		k[i] = key[i%Len];
	}
	for (i = 0; i < 256; i++)
	{
		j = (j + s[i] + k[i]) % 256;
		tmp = s[i];
		s[i] = s[j]; // 交换s[i]和s[j]
		s[j] = tmp;
	}
}

/*加解密*/
void rc4_crypt(unsigned char*s, unsigned char*Data, unsigned long Len)
{
	int i = 0, j = 0, t = 0;
	unsigned long k = 0;
	unsigned char tmp;
	for (k = 0; k < Len; k++)
	{
		i = (i + 1) % 256;
		j = (j + s[i]) % 256;
		tmp = s[i];
		s[i] = s[j]; // 交换s[x]和s[y]
		s[j] = tmp;
		t = (s[i] + s[j]) % 256;
		Data[k] ^= s[t];
	}
}

int main()
{
	unsigned char s[256] = { 0 }, s2[256] = { 0 }; // S-box
	char key[256] = { "12345678abcdefghijklmnopqrspxyz" };
	char pData[] = {0xbc,0xc5,0x12,0x7d,0x85,0x23,0x84,0x71,0x7b,0x39,0x28,0x2,0xd3,0x51,0xf3,0x2c,0x89,0x2b,0xa6,0x2c,0xaf,0x9,0x22,0x22};
	  
	unsigned long len = strlen(pData);
	int i;

	printf("pData=%s\n", pData);
	printf("key=%s,length=%d\n\n", key, strlen(key));
    
	rc4_init(s, (unsigned char*)key, strlen(key)); // 已经完成了初始化
	printf("\n\n");
	for (i = 0; i < 256; i++) // 用s2[i]暂时保留经过初始化的s[i]，很重要的！！！
	{
		s2[i] = s[i];
	}
    
    //可以看到，加解密函数都是相同的
	printf("已经加密，现在解密:\n\n");
	rc4_crypt(s2, (unsigned char*)pData, len); // 解密
	printf("pData=%s\n\n", pData);
	return 0;
}

key= 12345678abcdefghijklmnopqrspxyz