cvsroot/libcpjit/cpjit.C

/*
    cpjit.C -- c portable jit in time compiler
    Copyright (C) 2005      Marc Lehmann <cpjit@schmorp.de>
 
    This file is part of libcpjit.

    libcpjit is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation; either version 2 of the License, or
    (at your option) any later version.
 
    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.
 
    You should have received a copy of the GNU General Public License
    along with gvpe; if not, write to the Free Software
    Foundation, Inc. 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
*/

#include "config.h"

#include "cpjit.h"

#include <cassert>
#include <cstdlib>
#include <cstdarg>
#include <cstdio>
#include <cstring>
#include <cerrno>

#include <iostream>
#include <sstream>
#include <fstream>
#include <stdexcept>
#include <iomanip>

#include <unistd.h>
#include <fcntl.h> // for locking

#include <sys/stat.h>
#include <sys/types.h>
#include <dirent.h>
#include <utime.h>

#include "md5.h"

#include <dlfcn.h>

#define STR_(str) #str
#define STR(str) STR_(str)

#define IDX_VERSION 1

namespace cpjit
{

using namespace std;

const char typestr<U8   >::str[] = "unsigned " STR(TYPE_INT8);
const char typestr<U16  >::str[] = "unsigned " STR(TYPE_INT16);
const char typestr<U32  >::str[] = "unsigned " STR(TYPE_INT32);
const char typestr<U64  >::str[] = "unsigned " STR(TYPE_INT64);
const char typestr<I8   >::str[] = "signed "   STR(TYPE_INT8);
const char typestr<I16  >::str[] =             STR(TYPE_INT16);
const char typestr<I32  >::str[] =             STR(TYPE_INT32);
const char typestr<I64  >::str[] =             STR(TYPE_INT64);
const char typestr<FLT  >::str[] = "float";
const char typestr<DBL  >::str[] = "double";

const char typestr<PTR  >::str[] = "void"                      " *";

const char typestr<U8  *>::str[] = "unsigned " STR(TYPE_INT8)  " *";
const char typestr<U16 *>::str[] = "unsigned " STR(TYPE_INT16) " *";
const char typestr<U32 *>::str[] = "unsigned " STR(TYPE_INT32) " *";
const char typestr<U64 *>::str[] = "unsigned " STR(TYPE_INT64) " *";
const char typestr<I8  *>::str[] = "signed "   STR(TYPE_INT8)  " *";
const char typestr<I16 *>::str[] =             STR(TYPE_INT16) " *";
const char typestr<I32 *>::str[] =             STR(TYPE_INT32) " *";
const char typestr<I64 *>::str[] =             STR(TYPE_INT64) " *";
const char typestr<FLT *>::str[] = "float"                     " *";
const char typestr<DBL *>::str[] = "double"                    " *";

/////////////////////////////////////////////////////////////////////////////

struct digest : md5_ctx
{
  digest ();
  void operator ()(const string &data);
  operator string();
};

digest::digest ()
{
  md5_init_ctx (this);
}

void digest::operator ()(const string &data)
{
  md5_process_bytes (data.c_str (), data.size (), this);
}

digest::operator string ()
{
  unsigned char data[16];
  std::ostringstream s;

  md5_finish_ctx (this, data);

  for (int i = 0; i < 16; i++)
    s << std::setfill ('0') << std::hex << std::setw (2) << (int)data [i];

  return s.str ();
}

/////////////////////////////////////////////////////////////////////////////

static inline bool
is_null (const string &s)
{
  return s.empty ();
}

/////////////////////////////////////////////////////////////////////////////

// error exception builder
struct error : std::ostringstream
{
  ~error ()
  {
    throw std::runtime_error (str ());
  }
};

#define FATAL(msg) do { error o; o << msg << std::endl; } while (0)
#define ERRNO " (" << strerror (errno) << ")"

/////////////////////////////////////////////////////////////////////////////

static long get_stamp (int fd)
{
  struct stat st;

  if (fstat (fd, &st))
    FATAL ("could not get file stamp");

  return (long)st.st_mtime;
}

static void set_stamp (const string &path, long stamp)
{
  struct utimbuf ut;

  ut.actime  = stamp;
  ut.modtime = stamp;

  if (utime (path.c_str (), &ut))
    FATAL ("could not set file stamp");
}

env::env (const string &path, bool temporary)
: path(path), temporary(temporary)
{
  bool create;
  mode_t oflags = O_RDWR | O_EXCL;

  if (is_null (path))
    {
      temporary = true;
      this->path = tmpnam (0);
    }

  create = mkdir (this->path.c_str (), 0777) == 0;

  if (!create && temporary)
    FATAL ("error while creating libcpjit temporary directory" << ERRNO);
  
  idxpath = this->path + "/libcpjit.idx";
  idxfd = open (idxpath.c_str (), O_RDWR | (create ? O_CREAT | O_EXCL : 0), 0666);
  if (idxfd < 0)
    FATAL (path << ": unable to open index file " << idxpath);

  lock ();

  if (create)
    {
      idxstamp = get_stamp (idxfd) - 1;
      nextid = 0;
      save_index ();
    }

  load_index ();
  unlock ();
  compact ();
}

env::~env ()
{
  if (temporary)
    clear ();
}

void env::clear ()
{
  DIR *dir = opendir (path.c_str ());

  if (dir)
    {
      struct dirent *de;
      while ((de = readdir (dir)))
        {
          if (de->d_name [0] == '.')
            continue;

          string f = path + "/" + de->d_name;
          unlink (f.c_str ());
        }

      closedir (dir);
    }

  if (rmdir (path.c_str ()))
    FATAL (path << ": unable to remove libcpjit directory" << ERRNO);
}

void env::dolock (bool lock)
{
  struct flock fl;

  fl.l_type   = lock ? F_WRLCK : F_UNLCK;
  fl.l_whence = SEEK_SET;
  fl.l_start  = 0;
  fl.l_len    = 0;

  while (fcntl (idxfd, F_SETLKW, &fl) < 0)
    if (errno != EINTR)
      FATAL (path << ": unable to lock index file" << ERRNO);
}

void env::lock ()
{
  dolock (true);
}

void env::unlock ()
{
  dolock (false);
}

bool
env::load_index ()
{
  // caller must lock()

  long stamp = get_stamp (idxfd);

  if (stamp == idxstamp)
    return false;

  idxstamp = stamp;

  fstream f (idxpath.c_str (), ios::in);

  if (!f)
    FATAL (idxpath << ": unable to read index" << ERRNO);

  f >> nextid;

  if (nextid != IDX_VERSION)
    FATAL (idxpath << ": index file corrupted");

  f >> nextid;

  id2file.clear ();

  while (f)
    {
      string id; f >> id;

      if (id == "-")
        break;

      string fileid; f >> fileid;

      id2file [id] = fileid;

    }

  string end; f >> end;

  if (end != "__end__" || !f)
    FATAL (idxpath << ": index file corrupted");

  f.close ();

  return true;
}

void
env::save_index ()
{
  // caller must lock()+load_index()

  std::fstream f (idxpath.c_str ());

  if (!f)
    FATAL (idxpath << ": unable to write index" << ERRNO);

  f << IDX_VERSION << "\n"
    << nextid << "\n";

  for (map<string, string>::const_iterator i = id2file.begin ();
       i != id2file.end ();
       ++i)
    f << (*i).first << " " << (*i).second << "\n";

  f << "-\n";
  f << "__end__\n";

  f.close ();

  set_stamp (idxpath, ++idxstamp);
}

void
env::compact ()
{
  lock ();
  save_index ();
  unlock ();
}

void env::register_fragment (const string &id, const string &source)
{
  fragments.push_back (pair<string, string> (id, source));
}

void *env::dlsym (const string &id, const char *symbol)
{
  assert (id2file.find (id) != id2file.end ());

  const string &fileid = id2file [id];

  void *so;

  // load so
  if (file2so.find (fileid) == file2so.end ())
    {
      string sopath = path + "/" + fileid + ".so";

      so = dlopen (sopath.c_str (), RTLD_LAZY | RTLD_GLOBAL);
      if (!so)
        FATAL (sopath << ": so file should be there but isn't" << ERRNO);
    }
  else
    so = file2so [fileid];

  return ::dlsym (so, symbol);
}

void *env::sym (const string &id, const char *symbol)
{
  // not yet compiled?
  if (id2file.find (id) == id2file.end ())
    {
      lock ();

      // nobody else compiled it?
      if (!load_index () || id2file.find (id) == id2file.end ())
        {
          ostringstream sfile;
          sfile << nextid++;
          string fileid = sfile.str ();
          string base = path + "/" + fileid;

          string fc  = base + ".c";
          string fo  = base + ".o";
          string fso = base + ".so";

          std::ofstream f (fc.c_str ());

          for (vector< pair<string, string> >::const_iterator i = fragments.begin ();
               i != fragments.end ();
               ++i)
            if (id2file.find ((*i).first) == id2file.end ())
              {
                f << "#line 1 \"" << (*i).first << "\"\n"
                  << (*i).second << "\n";

                id2file [(*i).first] = fileid;
              }
          
          f.close ();
          fragments.clear ();

          system ((CC " " CFLAGS " -o " + fo + " " + fc).c_str ());
          unlink (fc.c_str ());
          system ((LD " " LFLAGS " -o " + fso + " " + fo).c_str ());

          save_index ();
        }

      unlock ();
    }

  void *p = dlsym (id, symbol);

  if (!p)
    {
      lock ();
      load_index ();
      unlock ();
      p = dlsym (id, symbol);
    }

  if (!p)
    FATAL ("symbol " << id << ":" << symbol << " not where it should be");

  return p;
}

/////////////////////////////////////////////////////////////////////////////

fun::fun (env &e, const string &id, const string &source)
: e(e), id(id), funptr(0)
{
  e.register_fragment (id, source);
}

void *fun::ptr ()
{
  if (!funptr)
    funptr = e.sym (id, id.c_str ());

  return funptr;
}

/////////////////////////////////////////////////////////////////////////////

funbuild::funbuild (env &e, const char *rettype, ...)
: e(e)
{
  retlist = rettype;

  std::ostringstream argl;

  va_list ap;
  va_start (ap, rettype);

  int args = 0;
  while ((rettype = va_arg (ap, const char *)))
    {
      if (args++)
        argl << ", ";

      argl << rettype << " a" << args;
    }

  va_end (ap);

  arglist = argl.str ();
}

void
funbuild::finish (string &id, string &source)
{
  digest dgst;

  dgst (retlist);
  dgst (arglist);
  dgst (str ());

  id = "f_" + (string) dgst;

  std::ostringstream o;
  o << retlist << " " << id << " (" << arglist << ")\n{\n" << str () << "\n}\n";
  source = o.str ();
}

}
Revision:	1.11
Committed:	Sat Oct 15 00:00:59 2005 UTC (18 years, 9 months ago) by root
Content type:	text/plain
Branch:	MAIN
Changes since 1.10:	+3 -16 lines
Log Message:	* empty log message *
#	Content
1	/*
2	cpjit.C -- c portable jit in time compiler
3	Copyright (C) 2005 Marc Lehmann <cpjit@schmorp.de>
4
5	This file is part of libcpjit.
6
7	libcpjit is free software; you can redistribute it and/or modify
8	it under the terms of the GNU General Public License as published by
9	the Free Software Foundation; either version 2 of the License, or
10	(at your option) any later version.
11
12	This program is distributed in the hope that it will be useful,
13	but WITHOUT ANY WARRANTY; without even the implied warranty of
14	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15	GNU General Public License for more details.
16
17	You should have received a copy of the GNU General Public License
18	along with gvpe; if not, write to the Free Software
19	Foundation, Inc. 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20	*/
21
22	#include "config.h"
23
24	#include "cpjit.h"
25
26	#include <cassert>
27	#include <cstdlib>
28	#include <cstdarg>
29	#include <cstdio>
30	#include <cstring>
31	#include <cerrno>
32
33	#include <iostream>
34	#include <sstream>
35	#include <fstream>
36	#include <stdexcept>
37	#include <iomanip>
38
39	#include <unistd.h>
40	#include <fcntl.h> // for locking
41
42	#include <sys/stat.h>
43	#include <sys/types.h>
44	#include <dirent.h>
45	#include <utime.h>
46
47	#include "md5.h"
48
49	#include <dlfcn.h>
50
51	#define STR_(str) #str
52	#define STR(str) STR_(str)
53
54	#define IDX_VERSION 1
55
56	namespace cpjit
57	{
58
59	using namespace std;
60
61	const char typestr<U8 >::str[] = "unsigned " STR(TYPE_INT8);
62	const char typestr<U16 >::str[] = "unsigned " STR(TYPE_INT16);
63	const char typestr<U32 >::str[] = "unsigned " STR(TYPE_INT32);
64	const char typestr<U64 >::str[] = "unsigned " STR(TYPE_INT64);
65	const char typestr<I8 >::str[] = "signed " STR(TYPE_INT8);
66	const char typestr<I16 >::str[] = STR(TYPE_INT16);
67	const char typestr<I32 >::str[] = STR(TYPE_INT32);
68	const char typestr<I64 >::str[] = STR(TYPE_INT64);
69	const char typestr<FLT >::str[] = "float";
70	const char typestr<DBL >::str[] = "double";
71
72	const char typestr<PTR >::str[] = "void" " *";
73
74	const char typestr<U8 >::str[] = "unsigned " STR(TYPE_INT8) " ";
75	const char typestr<U16 >::str[] = "unsigned " STR(TYPE_INT16) " ";
76	const char typestr<U32 >::str[] = "unsigned " STR(TYPE_INT32) " ";
77	const char typestr<U64 >::str[] = "unsigned " STR(TYPE_INT64) " ";
78	const char typestr<I8 >::str[] = "signed " STR(TYPE_INT8) " ";
79	const char typestr<I16 >::str[] = STR(TYPE_INT16) " ";
80	const char typestr<I32 >::str[] = STR(TYPE_INT32) " ";
81	const char typestr<I64 >::str[] = STR(TYPE_INT64) " ";
82	const char typestr<FLT >::str[] = "float" " ";
83	const char typestr<DBL >::str[] = "double" " ";
84
85	/////////////////////////////////////////////////////////////////////////////
86
87	struct digest : md5_ctx
88	{
89	digest ();
90	void operator ()(const string &data);
91	operator string();
92	};
93
94	digest::digest ()
95	{
96	md5_init_ctx (this);
97	}
98
99	void digest::operator ()(const string &data)
100	{
101	md5_process_bytes (data.c_str (), data.size (), this);
102	}
103
104	digest::operator string ()
105	{
106	unsigned char data[16];
107	std::ostringstream s;
108
109	md5_finish_ctx (this, data);
110
111	for (int i = 0; i < 16; i++)
112	s << std::setfill ('0') << std::hex << std::setw (2) << (int)data [i];
113
114	return s.str ();
115	}
116
117	/////////////////////////////////////////////////////////////////////////////
118
119	static inline bool
120	is_null (const string &s)
121	{
122	return s.empty ();
123	}
124
125	/////////////////////////////////////////////////////////////////////////////
126
127	// error exception builder
128	struct error : std::ostringstream
129	{
130	~error ()
131	{
132	throw std::runtime_error (str ());
133	}
134	};
135
136	#define FATAL(msg) do { error o; o << msg << std::endl; } while (0)
137	#define ERRNO " (" << strerror (errno) << ")"
138
139	/////////////////////////////////////////////////////////////////////////////
140
141	static long get_stamp (int fd)
142	{
143	struct stat st;
144
145	if (fstat (fd, &st))
146	FATAL ("could not get file stamp");
147
148	return (long)st.st_mtime;
149	}
150
151	static void set_stamp (const string &path, long stamp)
152	{
153	struct utimbuf ut;
154
155	ut.actime = stamp;
156	ut.modtime = stamp;
157
158	if (utime (path.c_str (), &ut))
159	FATAL ("could not set file stamp");
160	}
161
162	env::env (const string &path, bool temporary)
163	: path(path), temporary(temporary)
164	{
165	bool create;
166	mode_t oflags = O_RDWR \| O_EXCL;
167
168	if (is_null (path))
169	{
170	temporary = true;
171	this->path = tmpnam (0);
172	}
173
174	create = mkdir (this->path.c_str (), 0777) == 0;
175
176	if (!create && temporary)
177	FATAL ("error while creating libcpjit temporary directory" << ERRNO);
178
179	idxpath = this->path + "/libcpjit.idx";
180	idxfd = open (idxpath.c_str (), O_RDWR \| (create ? O_CREAT \| O_EXCL : 0), 0666);
181	if (idxfd < 0)
182	FATAL (path << ": unable to open index file " << idxpath);
183
184	lock ();
185
186	if (create)
187	{
188	idxstamp = get_stamp (idxfd) - 1;
189	nextid = 0;
190	save_index ();
191	}
192
193	load_index ();
194	unlock ();
195	compact ();
196	}
197
198	env::~env ()
199	{
200	if (temporary)
201	clear ();
202	}
203
204	void env::clear ()
205	{
206	DIR *dir = opendir (path.c_str ());
207
208	if (dir)
209	{
210	struct dirent *de;
211	while ((de = readdir (dir)))
212	{
213	if (de->d_name [0] == '.')
214	continue;
215
216	string f = path + "/" + de->d_name;
217	unlink (f.c_str ());
218	}
219
220	closedir (dir);
221	}
222
223	if (rmdir (path.c_str ()))
224	FATAL (path << ": unable to remove libcpjit directory" << ERRNO);
225	}
226
227	void env::dolock (bool lock)
228	{
229	struct flock fl;
230
231	fl.l_type = lock ? F_WRLCK : F_UNLCK;
232	fl.l_whence = SEEK_SET;
233	fl.l_start = 0;
234	fl.l_len = 0;
235
236	while (fcntl (idxfd, F_SETLKW, &fl) < 0)
237	if (errno != EINTR)
238	FATAL (path << ": unable to lock index file" << ERRNO);
239	}
240
241	void env::lock ()
242	{
243	dolock (true);
244	}
245
246	void env::unlock ()
247	{
248	dolock (false);
249	}
250
251	bool
252	env::load_index ()
253	{
254	// caller must lock()
255
256	long stamp = get_stamp (idxfd);
257
258	if (stamp == idxstamp)
259	return false;
260
261	idxstamp = stamp;
262
263	fstream f (idxpath.c_str (), ios::in);
264
265	if (!f)
266	FATAL (idxpath << ": unable to read index" << ERRNO);
267
268	f >> nextid;
269
270	if (nextid != IDX_VERSION)
271	FATAL (idxpath << ": index file corrupted");
272
273	f >> nextid;
274
275	id2file.clear ();
276
277	while (f)
278	{
279	string id; f >> id;
280
281	if (id == "-")
282	break;
283
284	string fileid; f >> fileid;
285
286	id2file [id] = fileid;
287
288	}
289
290	string end; f >> end;
291
292	if (end != "__end__" \|\| !f)
293	FATAL (idxpath << ": index file corrupted");
294
295	f.close ();
296
297	return true;
298	}
299
300	void
301	env::save_index ()
302	{
303	// caller must lock()+load_index()
304
305	std::fstream f (idxpath.c_str ());
306
307	if (!f)
308	FATAL (idxpath << ": unable to write index" << ERRNO);
309
310	f << IDX_VERSION << "\n"
311	<< nextid << "\n";
312
313	for (map<string, string>::const_iterator i = id2file.begin ();
314	i != id2file.end ();
315	++i)
316	f << (i).first << " " << (i).second << "\n";
317
318	f << "-\n";
319	f << "__end__\n";
320
321	f.close ();
322
323	set_stamp (idxpath, ++idxstamp);
324	}
325
326	void
327	env::compact ()
328	{
329	lock ();
330	save_index ();
331	unlock ();
332	}
333
334	void env::register_fragment (const string &id, const string &source)
335	{
336	fragments.push_back (pair<string, string> (id, source));
337	}
338
339	void env::dlsym (const string &id, const char symbol)
340	{
341	assert (id2file.find (id) != id2file.end ());
342
343	const string &fileid = id2file [id];
344
345	void *so;
346
347	// load so
348	if (file2so.find (fileid) == file2so.end ())
349	{
350	string sopath = path + "/" + fileid + ".so";
351
352	so = dlopen (sopath.c_str (), RTLD_LAZY \| RTLD_GLOBAL);
353	if (!so)
354	FATAL (sopath << ": so file should be there but isn't" << ERRNO);
355	}
356	else
357	so = file2so [fileid];
358
359	return ::dlsym (so, symbol);
360	}
361
362	void env::sym (const string &id, const char symbol)
363	{
364	// not yet compiled?
365	if (id2file.find (id) == id2file.end ())
366	{
367	lock ();
368
369	// nobody else compiled it?
370	if (!load_index () \|\| id2file.find (id) == id2file.end ())
371	{
372	ostringstream sfile;
373	sfile << nextid++;
374	string fileid = sfile.str ();
375	string base = path + "/" + fileid;
376
377	string fc = base + ".c";
378	string fo = base + ".o";
379	string fso = base + ".so";
380
381	std::ofstream f (fc.c_str ());
382
383	for (vector< pair<string, string> >::const_iterator i = fragments.begin ();
384	i != fragments.end ();
385	++i)
386	if (id2file.find ((*i).first) == id2file.end ())
387	{
388	f << "#line 1 \"" << (*i).first << "\"\n"
389	<< (*i).second << "\n";
390
391	id2file [(*i).first] = fileid;
392	}
393
394	f.close ();
395	fragments.clear ();
396
397	system ((CC " " CFLAGS " -o " + fo + " " + fc).c_str ());
398	unlink (fc.c_str ());
399	system ((LD " " LFLAGS " -o " + fso + " " + fo).c_str ());
400
401	save_index ();
402	}
403
404	unlock ();
405	}
406
407	void *p = dlsym (id, symbol);
408
409	if (!p)
410	{
411	lock ();
412	load_index ();
413	unlock ();
414	p = dlsym (id, symbol);
415	}
416
417	if (!p)
418	FATAL ("symbol " << id << ":" << symbol << " not where it should be");
419
420	return p;
421	}
422
423	/////////////////////////////////////////////////////////////////////////////
424
425	fun::fun (env &e, const string &id, const string &source)
426	: e(e), id(id), funptr(0)
427	{
428	e.register_fragment (id, source);
429	}
430
431	void *fun::ptr ()
432	{
433	if (!funptr)
434	funptr = e.sym (id, id.c_str ());
435
436	return funptr;
437	}
438
439	/////////////////////////////////////////////////////////////////////////////
440
441	funbuild::funbuild (env &e, const char *rettype, ...)
442	: e(e)
443	{
444	retlist = rettype;
445
446	std::ostringstream argl;
447
448	va_list ap;
449	va_start (ap, rettype);
450
451	int args = 0;
452	while ((rettype = va_arg (ap, const char *)))
453	{
454	if (args++)
455	argl << ", ";
456
457	argl << rettype << " a" << args;
458	}
459
460	va_end (ap);
461
462	arglist = argl.str ();
463	}
464
465	void
466	funbuild::finish (string &id, string &source)
467	{
468	digest dgst;
469
470	dgst (retlist);
471	dgst (arglist);
472	dgst (str ());
473
474	id = "f_" + (string) dgst;
475
476	std::ostringstream o;
477	o << retlist << " " << id << " (" << arglist << ")\n{\n" << str () << "\n}\n";
478	source = o.str ();
479	}
480
481	}