Mailing List Archive: [patch 01/10] Generate headers with arch-specific structs.

[patch 01/10] Generate headers with arch-specific structs.

Dec 1, 2006, 8:02 AM

Post #1 of 7 (124 views)

This patch adds a script to generate headers with arch-specific
structs which can be included on any architecture. Can be used
to deal with structs of "foreign" architectures, needed for
32-on-64 support for example.

Signed-off-by: Gerd Hoffmann <kraxel@suse.de>
---
tools/Rules.mk | 2
xen/Makefile | 5
xen/include/public/foreign/Makefile | 30 +++++
xen/include/public/foreign/mkchecker.py | 58 +++++++++++
xen/include/public/foreign/mkheader.py | 153 ++++++++++++++++++++++++++++++
xen/include/public/foreign/reference.size | 17 +++
xen/include/public/foreign/structs.py | 49 +++++++++
7 files changed, 314 insertions(+)

Index: build-32-unstable-12502/xen/include/public/foreign/Makefile
===================================================================
--- /dev/null
+++ build-32-unstable-12502/xen/include/public/foreign/Makefile
@@ -0,0 +1,30 @@
+XEN_ROOT := ../../../..
+include $(XEN_ROOT)/tools/Rules.mk
+
+architectures := x86_32 x86_64 ia64
+headers := $(patsubst %, %.h, $(architectures))
+scripts := $(wildcard *.py)
+
+.PHONY: all clean check-headers
+all: $(headers) check-headers
+
+clean:
+ rm -f $(headers)
+ rm -f checker checker.c $(XEN_TARGET_ARCH).size
+ rm -f *.pyc *.o *~
+
+check-headers: checker
+ifeq ($(CROSS_COMPILE),)
+ ./checker > $(XEN_TARGET_ARCH).size
+ diff -u reference.size $(XEN_TARGET_ARCH).size
+else
+ @echo "cross build: skipping check"
+endif
+
+%.h: ../arch-%.h ../xen.h $(scripts)
+ python mkheader.py $* $@ $< ../xen.h
+
+checker.o: checker.c $(headers)
+
+checker.c: $(scripts)
+ python mkchecker.py $(XEN_TARGET_ARCH) $@ $(architectures)
Index: build-32-unstable-12502/xen/include/public/foreign/mkheader.py
===================================================================
--- /dev/null
+++ build-32-unstable-12502/xen/include/public/foreign/mkheader.py
@@ -0,0 +1,153 @@
+#!/usr/bin/python
+
+import sys, re;
+from structs import structs, defines;
+
+# command line arguments
+arch = sys.argv[1];
+outfile = sys.argv[2];
+infiles = sys.argv[3:];
+
+
+###########################################################################
+
+###########################################################################
+# configuration #2: architecture information
+
+inttypes = {};
+header = {};
+footer = {};
+
+# x86_32
+inttypes["x86_32"] = {
+ "unsigned long" : "uint32_t",
+ "long" : "uint32_t",
+ "xen_pfn_t" : "uint32_t",
+};
+header["x86_32"] = """
+#pragma pack(push, 4)
+""";
+footer["x86_32"] = """
+#pragma pack(pop)
+""";
+
+# x86_64
+inttypes["x86_64"] = {
+ "unsigned long" : "__align8__ uint64_t",
+ "long" : "__align8__ uint64_t",
+ "xen_pfn_t" : "__align8__ uint64_t",
+};
+header["x86_64"] = """
+#ifdef __GNUC__
+# define __DECL_REG(name) union { uint64_t r ## name, e ## name; }
+# define __align8__ __attribute__((aligned (8)))
+#else
+# define __DECL_REG(name) uint64_t r ## name
+# define __align8__ FIXME
+#endif
+""";
+
+# ia64
+inttypes["ia64"] = {
+ "unsigned long" : "__align8__ uint64_t",
+ "long" : "__align8__ uint64_t",
+ "xen_pfn_t" : "__align8__ uint64_t",
+ "long double" : "__align16__ ldouble_t",
+};
+header["ia64"] = """
+#define __align8__ __attribute__((aligned (8)))
+#define __align16__ __attribute__((aligned (16)))
+typedef unsigned char ldouble_t[16];
+""";
+
+
+###########################################################################
+# main
+
+input = "";
+output = "";
+fileid = re.sub("[-.]", "_", "__FOREIGN_%s__" % outfile.upper());
+
+# read input header files
+for name in infiles:
+ f = open(name, "r");
+ input += f.read();
+ f.close();
+
+# add header
+output += """
+/*
+ * public xen defines and struct for %s
+ * generated by %s -- DO NOT EDIT
+ */
+
+#ifndef %s
+#define %s 1
+
+""" % (arch, sys.argv[0], fileid, fileid)
+
+if arch in header:
+ output += header[arch];
+ output += "\n";
+
+# add defines to output
+for line in re.findall("#define[^\n]+", input):
+ for define in defines:
+ regex = "#define\s+%s\\b" % define;
+ match = re.search(regex, line);
+ if None == match:
+ continue;
+ if define.upper()[0] == define[0]:
+ replace = define + "_" + arch.upper();
+ else:
+ replace = define + "_" + arch;
+ regex = "\\b%s\\b" % define;
+ output += re.sub(regex, replace, line) + "\n";
+output += "\n";
+
+# delete defines, comments, empty lines
+input = re.sub("#define[^\n]+\n", "", input);
+input = re.compile("/\*(.*?)\*/", re.S).sub("", input)
+input = re.compile("\n\s*\n", re.S).sub("\n", input);
+
+# add structs to output
+for struct in structs:
+ regex = "struct\s+%s\s*\{(.*?)\};" % struct;
+ match = re.search("struct\s+" + struct + "\s*\{(.*?)\};", input, re.S)
+ if None == match:
+ output += "#define %s_has_no_%s 1\n" % (arch, struct);
+ else:
+ output += "struct %s_%s {%s};\n" % (struct, arch, match.group(1));
+ output += "typedef struct %s_%s %s_%s_t;\n" % (struct, arch, struct, arch);
+ output += "\n";
+
+# add footer
+if arch in footer:
+ output += footer[arch];
+ output += "\n";
+output += "#endif /* %s */\n" % fileid;
+
+# replace: defines
+for define in defines:
+ if define.upper()[0] == define[0]:
+ replace = define + "_" + arch.upper();
+ else:
+ replace = define + "_" + arch;
+ output = re.sub("\\b%s\\b" % define, replace, output);
+
+# replace: structs + struct typedefs
+for struct in structs:
+ output = re.sub("\\b(struct\s+%s)\\b" % struct, "\\1_%s" % arch, output);
+ output = re.sub("\\b(%s)_t\\b" % struct, "\\1_%s_t" % arch, output);
+
+# replace: integer types
+integers = inttypes[arch].keys();
+integers.sort(lambda a, b: cmp(len(b),len(a)));
+for type in integers:
+ output = re.sub("\\b%s\\b" % type, inttypes[arch][type], output);
+
+# print results
+f = open(outfile, "w");
+f.write(output);
+f.close;
+
Index: build-32-unstable-12502/xen/include/public/foreign/structs.py
===================================================================
--- /dev/null
+++ build-32-unstable-12502/xen/include/public/foreign/structs.py
@@ -0,0 +1,49 @@
+# configuration: what needs translation
+
+structs = [. "start_info",
+ "trap_info",
+ "pt_fpreg",
+ "cpu_user_regs",
+ "xen_ia64_boot_param",
+ "ia64_tr_entry",
+ "vcpu_extra_regs",
+ "vcpu_guest_context",
+ "arch_vcpu_info",
+ "vcpu_time_info",
+ "vcpu_info",
+ "arch_shared_info",
+ "shared_info" ];
+
+defines = [. "FLAT_RING1_CS",
+ "FLAT_RING1_DS",
+ "FLAT_RING1_SS",
+
+ "FLAT_RING3_CS64",
+ "FLAT_RING3_DS64",
+ "FLAT_RING3_SS64",
+ "FLAT_KERNEL_CS64",
+ "FLAT_KERNEL_DS64",
+ "FLAT_KERNEL_SS64",
+
+ "FLAT_KERNEL_CS",
+ "FLAT_KERNEL_DS",
+ "FLAT_KERNEL_SS",
+
+ # x86_{32,64}
+ "_VGCF_i387_valid",
+ "VGCF_i387_valid",
+ "_VGCF_in_kernel",
+ "VGCF_in_kernel",
+ "_VGCF_failsafe_disables_events",
+ "VGCF_failsafe_disables_events",
+ "_VGCF_syscall_disables_events",
+ "VGCF_syscall_disables_events",
+
+ # ia64
+ "VGCF_EXTRA_REGS",
+
+ # all archs
+ "xen_pfn_to_cr3",
+ "MAX_VIRT_CPUS",
+ "MAX_GUEST_CMDLINE" ];
+
Index: build-32-unstable-12502/xen/include/public/foreign/reference.size
===================================================================
--- /dev/null
+++ build-32-unstable-12502/xen/include/public/foreign/reference.size
@@ -0,0 +1,17 @@
+
+structs | x86_32 x86_64 ia64
+
+start_info | 1104 1152 1152
+trap_info | 8 16 -
+pt_fpreg | - - 16
+cpu_user_regs | 68 200 496
+xen_ia64_boot_param | - - 96
+ia64_tr_entry | - - 32
+vcpu_extra_regs | - - 536
+vcpu_guest_context | 2800 5168 1056
+arch_vcpu_info | 24 16 0
+vcpu_time_info | 32 32 32
+vcpu_info | 64 64 48
+arch_shared_info | 268 280 272
+shared_info | 2584 3368 4384
+
Index: build-32-unstable-12502/xen/Makefile
===================================================================
--- build-32-unstable-12502.orig/xen/Makefile
+++ build-32-unstable-12502/xen/Makefile
@@ -31,8 +31,11 @@ _install: $(TARGET).gz
$(INSTALL_DATA) $(TARGET)-syms $(DESTDIR)/boot/$(notdir $(TARGET))-syms-$(XEN_FULLVERSION)
[ -d $(DESTDIR)/usr/include/xen/io ] || \
$(INSTALL_DIR) $(DESTDIR)/usr/include/xen/io
+ [ -d $(DESTDIR)/usr/include/xen/foreign ] || \
+ $(INSTALL_DIR) $(DESTDIR)/usr/include/xen/foreign
$(INSTALL_DATA) include/public/*.h $(DESTDIR)/usr/include/xen
$(INSTALL_DATA) include/public/io/*.h $(DESTDIR)/usr/include/xen/io
+ $(INSTALL_DATA) include/public/foreign/*.h $(DESTDIR)/usr/include/xen/foreign
$(INSTALL_DATA) include/public/COPYING $(DESTDIR)/usr/include/xen

.PHONY: _debug
@@ -42,6 +45,7 @@ _debug:
.PHONY: _clean
_clean: delete-unfresh-files
$(MAKE) -C tools clean
+ $(MAKE) -C include/public/foreign clean
$(MAKE) -f $(BASEDIR)/Rules.mk -C common clean
$(MAKE) -f $(BASEDIR)/Rules.mk -C drivers clean
$(MAKE) -f $(BASEDIR)/Rules.mk -C acm clean
@@ -60,6 +64,7 @@ $(TARGET).gz: $(TARGET)

$(TARGET): delete-unfresh-files
$(MAKE) -C tools
+ $(MAKE) -C include/public/foreign
$(MAKE) -f $(BASEDIR)/Rules.mk include/xen/compile.h
$(MAKE) -f $(BASEDIR)/Rules.mk include/xen/acm_policy.h
[ -e include/asm ] || ln -sf asm-$(TARGET_ARCH) include/asm
Index: build-32-unstable-12502/xen/include/public/foreign/mkchecker.py
===================================================================
--- /dev/null
+++ build-32-unstable-12502/xen/include/public/foreign/mkchecker.py
@@ -0,0 +1,58 @@
+#!/usr/bin/python
+
+import sys;
+from structs import structs;
+
+# command line arguments
+arch = sys.argv[1];
+outfile = sys.argv[2];
+archs = sys.argv[3:];
+
+f = open(outfile, "w");
+f.write('''
+/*
+ * sanity checks for generated foreign headers:
+ * - verify struct sizes
+ *
+ * generated by %s -- DO NOT EDIT
+ */
+#include <stdio.h>
+#include <stdlib.h>
+#include <stddef.h>
+#include <inttypes.h>
+#include "../xen.h"
+''');
+
+for a in archs:
+ f.write('#include "%s.h"\n' % a);
+
+f.write('int main(int argc, char *argv[])\n{\n');
+
+f.write('\tprintf("\\n");');
+f.write('printf("%-20s |", "structs");\n');
+for a in archs:
+ f.write('\tprintf("%%8s", "%s");\n' % a);
+f.write('\tprintf("\\n");');
+
+f.write('\tprintf("\\n");');
+for struct in structs:
+ f.write('\tprintf("%%-20s |", "%s");\n' % struct);
+ for a in archs:
+ if a == arch:
+ s = struct; # native
+ else:
+ s = struct + "_" + a;
+ f.write('#ifdef %s_has_no_%s\n' % (a, struct));
+ f.write('\tprintf("%8s", "-");\n');
+ f.write("#else\n");
+ f.write('\tprintf("%%8zd", sizeof(struct %s));\n' % s);
+ f.write("#endif\n");
+
+ f.write('\tprintf("\\n");\n\n');
+
+f.write('\tprintf("\\n");\n');
+f.write('\texit(0);\n');
+f.write('}\n');
+
+f.close();
+
Index: build-32-unstable-12502/tools/Rules.mk
===================================================================
--- build-32-unstable-12502.orig/tools/Rules.mk
+++ build-32-unstable-12502/tools/Rules.mk
@@ -46,5 +46,7 @@ mk-symlinks-xen:
( cd xen/hvm && ln -sf ../../$(XEN_ROOT)/xen/include/public/hvm/*.h . )
mkdir -p xen/io
( cd xen/io && ln -sf ../../$(XEN_ROOT)/xen/include/public/io/*.h . )
+ mkdir -p xen/foreign
+ ( cd xen/foreign && ln -sf ../../$(XEN_ROOT)/xen/include/public/foreign/*.h . )

mk-symlinks: mk-symlinks-xen mk-symlinks-$(XEN_OS)

--

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xensource.com
http://lists.xensource.com/xen-devel

[patch 01/10] Generate headers with arch-specific structs. [ In reply to ]

kraxel at suse

Dec 4, 2006, 3:58 AM

Post #2 of 7 (127 views)

Permalink

Re: [patch 01/10] Generate headers with arch-specific structs. [ In reply to ]

hollisb at us

Dec 4, 2006, 9:28 AM

Post #3 of 7 (122 views)

Permalink

On Mon, 2006-12-04 at 11:58 +0100, Gerd Hoffmann wrote:
> plain text document attachment (xen-generate-foreign-headers.diff)
> This patch adds a script to generate headers with arch-specific
> structs which can be included on any architecture. Can be used
> to deal with structs of "foreign" architectures, needed for
> 32-on-64 support for example.

This omits PowerPC; is that because we've made sure not to use
variable-sized types?

Even if that's the case, shouldn't we still have headers in
xen/include/public/foreign, since otherwise external tools #including
those headers will break?

--
Hollis Blanchard
IBM Linux Technology Center

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xensource.com
http://lists.xensource.com/xen-devel

Re: [patch 01/10] Generate headers with arch-specific structs. [ In reply to ]

kraxel at suse

Dec 4, 2006, 10:04 AM

Post #4 of 7 (122 views)

Permalink

Hollis Blanchard wrote:
> On Mon, 2006-12-04 at 11:58 +0100, Gerd Hoffmann wrote:
>> plain text document attachment (xen-generate-foreign-headers.diff)
>> This patch adds a script to generate headers with arch-specific
>> structs which can be included on any architecture. Can be used
>> to deal with structs of "foreign" architectures, needed for
>> 32-on-64 support for example.
>
> This omits PowerPC; is that because we've made sure not to use
> variable-sized types?

No. It's because (a) I don't have a working cross compiler and (b)
because it's the only bigendian architecture, so just generating those
headers isn't enough to have any other (xen-supported) architectures
access powerpc structs correctly. So I decided to just leave it as-is
for now. It certaily can be added, feel free to send patches ;)

> Even if that's the case, shouldn't we still have headers in
> xen/include/public/foreign, since otherwise external tools #including
> those headers will break?

Tools don't need them as long as the code is compiled native only. The
(very incomplete) xc_dom_powerpc.c file of the new domain builder simply
includes xen.h and arch-ppc.h.

The rewritten domain builder uses the foreign headers for
xc_dom_{x86,ia64}.c, because these source files are compiled on all
three little endian architectures. A real practical use this has for
x86 only, for upcoming 32-on-64 support. Compiling ia64 is handy
sometimes as you'll easily notice at least some kinds of build failures
even without a ia64 box. There is no real use (yet?), although I can
think of some (poking informations out of ia64 suspend/core images on
x86 boxes for example).

[ see
http://www.suse.de/~kraxel/patches/unstable-hg12663-20061201-quilt/tools-domain-builder-core.diff
]

HTH,
Gerd

--
Gerd Hoffmann <kraxel@suse.de>
http://www.suse.de/~kraxel/julika-dora.jpeg

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xensource.com
http://lists.xensource.com/xen-devel

Re: [patch 01/10] Generate headers with arch-specific structs. [ In reply to ]

hollisb at us

Dec 4, 2006, 1:12 PM

Post #5 of 7 (122 views)

Permalink

On Mon, 2006-12-04 at 18:04 +0100, Gerd Hoffmann wrote:
>
> Hollis Blanchard wrote:
> > On Mon, 2006-12-04 at 11:58 +0100, Gerd Hoffmann wrote:
> >> plain text document attachment (xen-generate-foreign-headers.diff)
> >> This patch adds a script to generate headers with arch-specific
> >> structs which can be included on any architecture. Can be used
> >> to deal with structs of "foreign" architectures, needed for
> >> 32-on-64 support for example.
> >
> > This omits PowerPC; is that because we've made sure not to use
> > variable-sized types?
>
> No. It's because (a) I don't have a working cross compiler and (b)
> because it's the only bigendian architecture, so just generating those
> headers isn't enough to have any other (xen-supported) architectures
> access powerpc structs correctly. So I decided to just leave it as-is
> for now. It certaily can be added, feel free to send patches ;)

Hmm, that's ambitious. What exactly is your use case? If it's 32-on-64,
no endianness code is required, right?

--
Hollis Blanchard
IBM Linux Technology Center

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xensource.com
http://lists.xensource.com/xen-devel

Re: [patch 01/10] Generate headers with arch-specific structs. [ In reply to ]

kraxel at suse

Dec 5, 2006, 1:45 AM

Post #6 of 7 (122 views)

Permalink

Hollis Blanchard wrote:
>> No. It's because (a) I don't have a working cross compiler and (b)
>> because it's the only bigendian architecture, so just generating those
>> headers isn't enough to have any other (xen-supported) architectures
>> access powerpc structs correctly. So I decided to just leave it as-is
>> for now. It certaily can be added, feel free to send patches ;)
>
> Hmm, that's ambitious. What exactly is your use case? If it's 32-on-64,
> no endianness code is required, right?

That is the only one for now, yes, and it works fine without
byteswapping ...

cheers,
Gerd

--
Gerd Hoffmann <kraxel@suse.de>
http://www.suse.de/~kraxel/julika-dora.jpeg

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xensource.com
http://lists.xensource.com/xen-devel

Re: [patch 01/10] Generate headers with arch-specific structs. [ In reply to ]

hollisb at us

Dec 5, 2006, 6:52 AM

Post #7 of 7 (122 views)

Permalink

On Tue, 2006-12-05 at 09:45 +0100, Gerd Hoffmann wrote:
> Hollis Blanchard wrote:
> >> No. It's because (a) I don't have a working cross compiler and (b)
> >> because it's the only bigendian architecture, so just generating those
> >> headers isn't enough to have any other (xen-supported) architectures
> >> access powerpc structs correctly. So I decided to just leave it as-is
> >> for now. It certaily can be added, feel free to send patches ;)
> >
> > Hmm, that's ambitious. What exactly is your use case? If it's 32-on-64,
> > no endianness code is required, right?
>
> That is the only one for now, yes, and it works fine without
> byteswapping ...

So it should be fine that PowerPC structs don't have any endian-swapping
accessors.

--
Hollis Blanchard
IBM Linux Technology Center

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xensource.com
http://lists.xensource.com/xen-devel