vaudeville

# Description

Info:
The Dramatis Personae invite you to join in this latest production. nc vaudeville.chal.uiuc.tf 1337
File: vaudeville

TL;DR:
Analyze the binary => we can see that it will give us a number and we must return an integer which is the answer.
Analyze sub_3660(x, y) => Some sort of encrypting function.
Deal with functions like fork() and pipe() => create a child process to give the parent a string and save the length of it.
Goal: understanding sub_3660(x, y), write a decrypt script to find the input and submit to server to get the flag.

# Deep Analyze

Here’s the main function:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21

__int64 __fastcall main(__int64 a1, char **a2, char **a3)
{
  __int64 v3; // rbp
  int v4; // ebx
  unsigned int v6; // [rsp+4h] [rbp-24h] BYREF
  unsigned __int64 v7; // [rsp+8h] [rbp-20h]

  v7 = __readfsqword(0x28u);
  v3 = (unsigned int)sub_35F0(a1, a2, a3);
  __printf_chk(1LL, "Challenge: %u\n", v3);
  __printf_chk(1LL, "Response: ");
  fflush(stdout);
  __isoc99_scanf("%u", &v6);
  fflush(stdin);
  v4 = sub_3660(v6, 7);
  if ( v4 == (unsigned int)sub_3660(v3, 140) )
    sub_3DD0();
  else
    puts("that's rough buddy");
  return 0LL;
}

Static analyzing it, I know that v3 is the challenge’s number, then it reads our input v6 and do some encryption related to v3, if it satisfies the condition, we got the flag. So we have to dive into sub_3660(x, y)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25

  v2 = 0LL;
  v50 = __readfsqword(0x28u);
  v47 = 0;
  ptr[0] = (__int128)_mm_load_si128((const __m128i *)&xmmword_C450);
  ptr[1] = ptr[0];
  v45[0] = ptr[0];
  v45[1] = ptr[0];
  v46[0] = ptr[0];
  v46[1] = ptr[0];
  do
  {
    *((_BYTE *)v45 + v2) = ((a1 & (1 << v2)) != 0) + 48;
    ++v2;
  }
  while ( v2 != 32 );
  memset(filename, 0, sizeof(filename));
  __sprintf_chk(filename, 1LL, 128LL, "/tmp/tmp_%u", a1);
  stream = fopen(filename, "wb");
  setvbuf(stream, 0LL, 2, 0LL);
  fwrite(ptr, 1uLL, 0x60uLL, stream);
  fflush(stream);
  v3 = fopen(filename, "rb");
  v4 = fopen(filename, "rb");
  setvbuf(v3, 0LL, 2, 0LL);
  setvbuf(v4, 0LL, 2, 0LL);

First part of the function, it will pre-init data base on our input, I’ll call it data, and data looks like this: "0" * 32 + (binary form of our input in reverse) + "0" * 32. Then it creates folder /tmp/tmp_(our input) and write that pre-init data to it. We can easily check it by putting breakpoint after the fwrite() and cat /tmp/tmp_%u

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46

v39 = 0;
LABEL_5:
    fseek(v3, 32LL, 0);
    fseek(v4, 19LL, 0);
    v5 = v45;
    do
    {
      fread(&v40, 1uLL, 1uLL, v3);
      fread(&v41, 1uLL, 1uLL, v4);
      v6 = v41;
      v7 = v40;
      if ( pipe(&v42) )
      {
        fwrite("Pipe failed.\n", 1uLL, 0xDuLL, stderr);
        v11 = -1;
      }
      else
      {
        v8 = fork();
        if ( !v8 )
        {
          close(v42);
          v31 = dup(fd);
          v32 = fdopen(fd, "w");
          v33 = fdopen(v31, "w");
          fwrite(*(&off_E020 + v7), 1uLL, v7, v32);
          v34 = v6;
          v35 = *(&off_E020 + v6);
          goto LABEL_28;
        }
        if ( v8 < 0 )
        {
LABEL_33:
          fwrite("Fork failed.\n", 1uLL, 0xDuLL, stderr);
          exit(-1);
        }
        close(fd);
        v9 = fdopen(v42, "r");
        v10 = fread(v49, 1uLL, 0x200uLL, v9);
        fclose(v9);
        v11 = v10;
      }
      *(_BYTE *)v5 = v11;
      v5 = (__int128 *)((char *)v5 + 1);
    }
    while ( v46 != v5 );

This is a very interesting part in our functions:

  • First it reads data begin from positions 19 and 32, I’ll call it a and b.
  • It will create a communication pipe between parent and child process. The parent process will read what the child one give it to. Meanwhile the child will concat 2 strings from off_E020, and their indices are int(a) and int(b), then give it to the parent.
  • Finally, it takes the length of the concat string and store into data. Overall, this code blocks will replace each characters from 32 to 63 in our data using some encryption. To clarify informations, I inspect off_E020 to see if something is special, and indeed, there’s something.

If you look closely, you’ll see that it stores a 256 strings, sort by their lengths, from 0 to 255 :D. So basically the above code just take two chars, add them together and put it back to data.
Interestingly, the rest of the code in this function will encrypt the data similarly to previous code blocks, and it will loop a2 times - second argument in the function.
Now we know what this binary do: the challenge asks us to give a number so that this number after encrypt 7 times, will equals to the challenge’s number encrypted 140 times.

# Final Step

I wrote a small script that emulate the encryption part:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25

def encrypted(li):
    id1 = 32
    id2 = 19
    or_li = [i for i in li]
    for oo in range(32):
        li[id1] = (or_li[id1] + or_li[id2]) & 0xff
        id1 += 1
        id2 += 1

    id1 = 32
    id2 = 49
    or_li = [i for i in li]
    for oo in range(32):
        li[id1] = (or_li[id1] + or_li[id2]) & 0xff
        id1 += 1
        id2 += 1

    id1 = 32
    id2 = 27
    or_li = [i for i in li]
    for oo in range(32):
        li[id1] = (or_li[id1] + or_li[id2]) & 0xff
        id1 += 1
        id2 += 1
    return li

Now all we have to do is reverse this encryption. Here’s the decrypt script:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

def decrypted(li):
    id = 32
    id1 = 32
    id2 = 27
    for oo in range(32):
        li[id1] = (li[id1] - li[id2]) & 0xff
        id1 += 1
        id2 += 1

    id1 = 63
    id2 = 80
    for oo in range(32):
        li[id1] = (li[id1] - li[id2]) & 0xff
        id1 -= 1
        id2 -= 1

    id1 = 32
    id2 = 19
    for oo in range(32):
        li[id1] = (li[id1] - li[id2]) & 0xff
        id1 += 1
        id2 += 1

    return li

Put everything together, we have the script:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79

val = int(input())
# val = 1501756717

def convert(v):
    t = ''
    while v != 0:
        t += str(v & 1)
        v >>= 1
    t += '0' * (32 - len(t))
    return t

def encrypted(li):
    id1 = 32
    id2 = 19
    or_li = [i for i in li]
    for oo in range(32):
        li[id1] = (or_li[id1] + or_li[id2]) & 0xff
        id1 += 1
        id2 += 1

    id1 = 32
    id2 = 49
    or_li = [i for i in li]
    for oo in range(32):
        li[id1] = (or_li[id1] + or_li[id2]) & 0xff
        id1 += 1
        id2 += 1

    id1 = 32
    id2 = 27
    or_li = [i for i in li]
    for oo in range(32):
        li[id1] = (or_li[id1] + or_li[id2]) & 0xff
        id1 += 1
        id2 += 1
    return li

def decrypted(li):
    id = 32
    id1 = 32
    id2 = 27
    for oo in range(32):
        li[id1] = (li[id1] - li[id2]) & 0xff
        id1 += 1
        id2 += 1

    id1 = 63
    id2 = 80
    for oo in range(32):
        li[id1] = (li[id1] - li[id2]) & 0xff
        id1 -= 1
        id2 -= 1

    id1 = 32
    id2 = 19
    for oo in range(32):
        li[id1] = (li[id1] - li[id2]) & 0xff
        id1 += 1
        id2 += 1

    return li

payload = '0' * 32 + convert(val) + '0' * 32
li = [ord(i) for i in payload]
times = 7

for i in range(140):
    li = encrypted(li)

for i in range(7):
    li = decrypted(li)
de = li

ans = 0
id = 32
for i in range(32):
    ans += ((de[i + id] & 1) << i)

print(ans)

Submit the answer and we get the flag.
uiuctf{b0rne_4ward_c3as313ss1y_1nt0_teh_f-u-t-u-r-e_*dab*}