summaryrefslogtreecommitdiff
path: root/modules/pam_unix/bigcrypt.c
blob: e10d1c5605ac5626335bc3a58a96843ae8e7ac68 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
/*
 * This function implements the "bigcrypt" algorithm specifically for
 * Linux-PAM.
 *
 * This algorithm is algorithm 0 (default) shipped with the C2 secure
 * implementation of Digital UNIX.
 *
 * Disclaimer: This work is not based on the source code to Digital
 * UNIX, nor am I connected to Digital Equipment Corp, in any way
 * other than as a customer. This code is based on published
 * interfaces and reasonable guesswork.
 *
 * Description: The cleartext is divided into blocks of SEGMENT_SIZE=8
 * characters or less. Each block is encrypted using the standard UNIX
 * libc crypt function. The result of the encryption for one block
 * provides the salt for the suceeding block.
 *
 * Restrictions: The buffer used to hold the encrypted result is
 * statically allocated. (see MAX_PASS_LEN below).  This is necessary,
 * as the returned pointer points to "static data that are overwritten
 * by each call", (XPG3: XSI System Interface + Headers pg 109), and
 * this is a drop in replacement for crypt();
 *
 * Andy Phillips <atp@mssl.ucl.ac.uk>
 */

#include "config.h"

#include <string.h>
#include <stdlib.h>
#include <security/_pam_macros.h>
#ifdef HAVE_LIBXCRYPT
#include <xcrypt.h>
#elif defined(HAVE_CRYPT_H)
#include <crypt.h>
#endif

#include "bigcrypt.h"

/*
 * Max cleartext password length in segments of 8 characters this
 * function can deal with (16 segments of 8 chars= max 128 character
 * password).
 */

#define MAX_PASS_LEN       16
#define SEGMENT_SIZE       8
#define SALT_SIZE          2
#define KEYBUF_SIZE        ((MAX_PASS_LEN*SEGMENT_SIZE)+SALT_SIZE)
#define ESEGMENT_SIZE      11
#define CBUF_SIZE          ((MAX_PASS_LEN*ESEGMENT_SIZE)+SALT_SIZE+1)

char *bigcrypt(const char *key, const char *salt)
{
	char *dec_c2_cryptbuf;
#ifdef HAVE_CRYPT_R
	struct crypt_data *cdata;
#endif
	unsigned long int keylen, n_seg, j;
	char *cipher_ptr, *plaintext_ptr, *tmp_ptr, *salt_ptr;
	char keybuf[KEYBUF_SIZE + 1];

	D(("called with key='%s', salt='%s'.", key, salt));

	/* reset arrays */
	dec_c2_cryptbuf = malloc(CBUF_SIZE);
	if (!dec_c2_cryptbuf) {
		return NULL;
	}
#ifdef HAVE_CRYPT_R
	cdata = malloc(sizeof(*cdata));
	if(!cdata) {
		free(dec_c2_cryptbuf);
		return NULL;
	}
	cdata->initialized = 0;
#endif
	memset(keybuf, 0, KEYBUF_SIZE + 1);
	memset(dec_c2_cryptbuf, 0, CBUF_SIZE);

	/* fill KEYBUF_SIZE with key */
	strncpy(keybuf, key, KEYBUF_SIZE);

	/* deal with case that we are doing a password check for a
	   conventially encrypted password: the salt will be
	   SALT_SIZE+ESEGMENT_SIZE long. */
	if (strlen(salt) == (SALT_SIZE + ESEGMENT_SIZE))
		keybuf[SEGMENT_SIZE] = '\0';	/* terminate password early(?) */

	keylen = strlen(keybuf);

	if (!keylen) {
		n_seg = 1;
	} else {
		/* work out how many segments */
		n_seg = 1 + ((keylen - 1) / SEGMENT_SIZE);
	}

	if (n_seg > MAX_PASS_LEN)
		n_seg = MAX_PASS_LEN;	/* truncate at max length */

	/* set up some pointers */
	cipher_ptr = dec_c2_cryptbuf;
	plaintext_ptr = keybuf;

	/* do the first block with supplied salt */
#ifdef HAVE_CRYPT_R
	tmp_ptr = crypt_r(plaintext_ptr, salt, cdata);	/* libc crypt_r() */
#else
	tmp_ptr = crypt(plaintext_ptr, salt);	/* libc crypt() */
#endif
	/* and place in the static area */
	strncpy(cipher_ptr, tmp_ptr, 13);
	cipher_ptr += ESEGMENT_SIZE + SALT_SIZE;
	plaintext_ptr += SEGMENT_SIZE;	/* first block of SEGMENT_SIZE */

	/* change the salt (1st 2 chars of previous block) - this was found
	   by dowsing */

	salt_ptr = cipher_ptr - ESEGMENT_SIZE;

	/* so far this is identical to "return crypt(key, salt);", if
	   there is more than one block encrypt them... */

	if (n_seg > 1) {
		for (j = 2; j <= n_seg; j++) {

#ifdef HAVE_CRYPT_R
			tmp_ptr = crypt_r(plaintext_ptr, salt_ptr, cdata);
#else
			tmp_ptr = crypt(plaintext_ptr, salt_ptr);
#endif

			/* skip the salt for seg!=0 */
			strncpy(cipher_ptr, (tmp_ptr + SALT_SIZE), ESEGMENT_SIZE);

			cipher_ptr += ESEGMENT_SIZE;
			plaintext_ptr += SEGMENT_SIZE;
			salt_ptr = cipher_ptr - ESEGMENT_SIZE;
		}
	}
	D(("key=|%s|, salt=|%s|\nbuf=|%s|\n", key, salt, dec_c2_cryptbuf));

#ifdef HAVE_CRYPT_R
	free(cdata);
#endif

	/* this is the <NUL> terminated encrypted password */
	return dec_c2_cryptbuf;
}