[COMMIT osv master] tests: new pthread should inherit parent CPU affinity

[Commit Bot]

From: Justin Cinkelj 
Committer: Nadav Har'El 
Branch: master

tests: new pthread should inherit parent CPU affinity

New pthread should have CPU affinity requested be cpuset in
attributes. If explicit cpu pinning is not requested, than parent
thread CPU pinning should be used.

Signed-off-by: Justin Cinkelj 
Message-Id: <20161129141222.11595-5-justin.cink...@xlab.si>

---
diff --git a/modules/tests/Makefile b/modules/tests/Makefile
--- a/modules/tests/Makefile
+++ b/modules/tests/Makefile
@@ -84,7 +84,7 @@ tests := tst-pthread.so misc-ramdisk.so tst-vblk.so  
tst-bsd-evh.so \

tst-namespace.so tst-without-namespace.so payload-env.so \
 	payload-merge-env.so misc-execve.so misc-execve-payload.so misc-mutex2.so  
\

tst-pthread-setcancelstate.so tst-syscall.so tst-pin.so tst-run.so \
-   tst-ifaddrs.so
+   tst-ifaddrs.so tst-pthread-affinity-inherit.so

 #  libstatic-thread-variable.so tst-static-thread-variable.so \

diff --git a/tests/tst-pthread-affinity-inherit.cc  
b/tests/tst-pthread-affinity-inherit.cc

--- a/tests/tst-pthread-affinity-inherit.cc
+++ b/tests/tst-pthread-affinity-inherit.cc
@@ -0,0 +1,310 @@
+/*
+ * This work is open source software, licensed under the terms of the
+ * BSD license as described in the LICENSE file in the top-level directory.
+ */
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+static std::atomic tests_total(0), tests_failed(0);
+
+void report(const char* name, bool passed)
+{
+static const char* status[] = { "FAIL", "PASS" };
+printf("%s: %s\n", status[passed], name);
+tests_total += 1;
+tests_failed += !passed;
+}
+
+void *check_cpuset(void *ref_cs2)
+{
+const cpu_set_t *ref_cs = static_cast(ref_cs2);
+cpu_set_t cs;
+unsigned int cpu_id;
+bool success;
+
+sched_getaffinity(0, sizeof(cs), );
+success = true;
+for (size_t i = 0; i < sched::cpus.size(); i++) {
+success = success && (CPU_ISSET(i, ) == CPU_ISSET(i, ref_cs));
+}
+report("child pinning is equal to reference cpuset", success);
+
+cpu_id = sched::thread::current()->get_cpu()->id;
+report("started on the correct CPU", CPU_ISSET(cpu_id, ref_cs));
+
+usleep(1000);
+
+cpu_id = sched::thread::current()->get_cpu()->id;
+report("re-scheduled on the correct CPU", CPU_ISSET(cpu_id, ref_cs));
+
+return NULL;
+}
+
+int main(void)
+{
+printf("starting pthread affinity inherited from parent test\n");
+
+cpu_set_t cs, ref_cs;
+pthread_t thread;
+pthread_attr_t attr;
+bool success;
+pthread_t* threads;
+pthread_attr_t* attrs;
+cpu_set_t* ref_css;
+
+// we are started from httpserver, and should be unpinned.
+CPU_ZERO();
+sched_getaffinity(0, sizeof(cs), );
+success = CPU_COUNT() == (int)sched::cpus.size();
+report("main is pinned to exactly sched::cpus.size cpus", success);
+success = true;
+for (size_t i = 0; i < sched::cpus.size(); i++) {
+success = success && CPU_ISSET(i, );
+}
+report("main is pinned to all cpus", success);
+
+// new threads should be unpinned, as cpuset requests unpinned thread
+printf("new threads from unpinned parent sequential test:\n");
+CPU_ZERO();
+for (size_t i = 0; i < sched::cpus.size(); i++) {
+CPU_SET(i, );
+}
+for (size_t i = 0; i < sched::cpus.size(); i++) {
+pthread_attr_init();
+pthread_attr_setaffinity_np(, sizeof(cs), );
+pthread_create(, , check_cpuset, );
+pthread_join(thread, NULL);
+pthread_attr_destroy();
+}
+
+// new threads should be unpinned, because parent is unpinned
+printf("new threads from unpinned parent sequential test (attr with  
unset cpuset):\n");

+CPU_ZERO();
+for (size_t i = 0; i < sched::cpus.size(); i++) {
+CPU_SET(i, );
+}
+for (size_t i = 0; i < sched::cpus.size(); i++) {
+pthread_attr_init();
+pthread_create(, , check_cpuset, );
+pthread_join(thread, NULL);
+pthread_attr_destroy();
+}
+
+// new threads should be unpinned, because parent is unpinned
+printf("new threads from unpinned parent sequential test  
(attr=NULL):\n");

+CPU_ZERO();
+for (size_t i = 0; i < sched::cpus.size(); i++) {
+CPU_SET(i, );
+}
+for (size_t i = 0; i < sched::cpus.size(); i++) {
+pthread_create(, NULL, check_cpuset, );
+pthread_join(thread, NULL);
+}
+
+printf("new threads from unpinned parent parallel test:\n");
+threads = new pthread_t[sched::cpus.size()];
+attrs = new pthread_attr_t[sched::cpus.size()];
+CPU_ZERO();
+for (size_t i = 0; i < sched::cpus.size(); i++) {
+CPU_SET(i, );
+}
+for (size_t i = 0; i < sched::cpus.size(); i++) {
+pthread_attr_init([i]);
+pthread_attr_setaffinity_np([i], sizeof(cs), );
+}
+for (size_t i = 0; i < 

[COMMIT osv master] pthread: pthread_spin_lock must not disabled preemption

[Commit Bot]

From: Nadav Har'El 
Committer: Nadav Har'El 
Branch: master

pthread: pthread_spin_lock must not disabled preemption

Our pthread_spin_lock implementation incorrectly used the kernel spinlock
implementation which disabled preemption, which is both inappropriate and
unnecessary for application code.

Ironically, after commit 930cb83d914796893746d5d7f7f9cf6df37db5e6,
pthread_spin_lock() became completely unusable: Any appliction will call
pthread_spin_unlock() while the spin-lock is held, and the first call
will require a symbol resolution which asserts preemption is not disabled,
and crashes.

This patch fixes the pthread_spin_lock implementation to not disable
preemption.

This patch also adds a test for this case, which crashes before this patch
(and succeeds with it).

Fixes #814

Signed-off-by: Nadav Har'El 
Message-Id: <1480503933-8157-1-git-send-email-...@scylladb.com>

---
diff --git a/libc/pthread.cc b/libc/pthread.cc
--- a/libc/pthread.cc
+++ b/libc/pthread.cc
@@ -28,7 +28,6 @@
 #include 

 #include 
-#include 
 #include 

 #include "pthread.hh"
@@ -316,16 +315,24 @@ int pthread_getcpuclockid(pthread_t thread, clockid_t  
*clock_id)

 return 0;
 }

-// pthread_spinlock_t and spinlock_t aren't really the same type. But since
-// spinlock_t is a boolean and pthread_spinlock_t is defined to be an  
integer,
-// just casting it like this is fine. As long as we are never operating  
more

-// than sizeof(int) at a time, we should be fine.
+// Note that for pthread_spin_lock() we cannot use the implementation
+// from  because it disables preemption, which is
+// inappropriate for application code, and also unnecessary (the kernel
+// version needs to defend against a deadlock when one of the lock holders
+// disables preemption - but an application cannot disable preemption).
+// So we repeat similar code here.
+inline bool* from_libc(pthread_spinlock_t* a) {
+static_assert(sizeof(bool) <= sizeof(pthread_spinlock_t),
+  "pthread_spinlock_t cannot hold a bool");
+return reinterpret_cast(a);
+}
+
 int pthread_spin_init(pthread_spinlock_t *lock, int pshared)
 {
-static_assert(sizeof(spinlock_t) <= sizeof(pthread_spinlock_t),
-  "OSv spinlock type doesn't match pthread's");
-// PTHREAD_PROCESS_SHARED and PTHREAD_PROCESS_PRIVATE are the same  
while we have a single process.

-spinlock_init(reinterpret_cast(lock));
+// PTHREAD_PROCESS_SHARED and PTHREAD_PROCESS_PRIVATE are the same  
while

+// we have a single process.
+bool* b = from_libc(lock);
+*b = false;
 return 0;
 }

@@ -336,21 +343,29 @@ int pthread_spin_destroy(pthread_spinlock_t *lock)

 int pthread_spin_lock(pthread_spinlock_t *lock)
 {
-spin_lock(reinterpret_cast(lock));
+bool* b = from_libc(lock);
+while (__sync_lock_test_and_set(b, 1)) {
+while (*b) {
+barrier();
+// FIXME: use "PAUSE" instruction here
+}
+}
 return 0; // We can't really do deadlock detection
 }

 int pthread_spin_trylock(pthread_spinlock_t *lock)
 {
-if (!spin_trylock(reinterpret_cast(lock))) {
+bool* b = from_libc(lock);
+if (__sync_lock_test_and_set(b, 1)) {
 return EBUSY;
 }
 return 0;
 }

 int pthread_spin_unlock(pthread_spinlock_t *lock)
 {
-spin_unlock(reinterpret_cast(lock));
+bool* b = from_libc(lock);
+__sync_lock_release(b, 0);
 return 0;
 }

diff --git a/tests/tst-pthread.c b/tests/tst-pthread.c
--- a/tests/tst-pthread.c
+++ b/tests/tst-pthread.c
@@ -158,6 +158,20 @@ int main(void)
 printf("ts2 = %ld,%ld\n",ts2.tv_sec, ts2.tv_nsec);
 printf("ns = %ld\n",ns);

+// Test that pthread_spin_unlock() doesn't crash when it is resolved  
for

+// the first time while a spinlock is taken (see issue #814)
+pthread_spinlock_t spin;
+pthread_spin_init(, PTHREAD_PROCESS_PRIVATE);
+pthread_spin_lock();
+pthread_spin_unlock();
+pthread_spin_destroy();
+// Moreover, the application may even sleep while holding a spinlock
+pthread_spin_init(, PTHREAD_PROCESS_PRIVATE);
+pthread_spin_lock();
+usleep(1000);
+pthread_spin_unlock();
+pthread_spin_destroy();
+
 printf("SUMMARY: %u tests / %u failures\n", tests_total, tests_failed);
 return tests_failed == 0 ? 0 : 1;
 }

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] pthread_mutex_trylock: Return EBUSY not -EBUSY

[Commit Bot]

From: Rick Payne 
Committer: Nadav Har'El 
Branch: master

pthread_mutex_trylock: Return EBUSY not -EBUSY

Signed-off-by: Rick Payne 
Message-Id: <1480693088-22453-1-git-send-email-ri...@rossfell.co.uk>

---
diff --git a/libc/pthread.cc b/libc/pthread.cc
--- a/libc/pthread.cc
+++ b/libc/pthread.cc
@@ -430,7 +430,7 @@ int pthread_mutex_lock(pthread_mutex_t *m)
 int pthread_mutex_trylock(pthread_mutex_t *m)
 {
 if (!from_libc(m)->try_lock()) {
-return -EBUSY;
+return EBUSY;
 }
 return 0;
 }

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] sendfile: don't try to send beyond the end of file

[Commit Bot]

From: Rick Payne 
Committer: Nadav Har'El 
Branch: master

sendfile: don't try to send beyond the end of file

Ensure that sendfile doesn't try to send beyond the end of the source
file. Added some tests, and ensured that the same result is observed under
both Linux and OSv.

Fixes #815

Signed-off-by: Rick Payne 
Message-Id: <1480907640-15245-1-git-send-email-ri...@rossfell.co.uk>

---
diff --git a/fs/vfs/main.cc b/fs/vfs/main.cc
--- a/fs/vfs/main.cc
+++ b/fs/vfs/main.cc
@@ -2039,6 +2039,21 @@ int sendfile(int out_fd, int in_fd, off_t *_offset,  
size_t count)

 offset = lseek(in_fd, 0, SEEK_CUR);
 }

+// Constrain count to the extent of the file...
+struct stat st;
+if (fstat(in_fd, ) < 0) {
+return -1;
+} else {
+if (offset >= st.st_size) {
+return 0;
+} else if ((offset + count) >= st.st_size) {
+count = st.st_size - offset;
+if (count == 0) {
+return 0;
+}
+}
+}
+
 size_t bytes_to_mmap = count + (offset % mmu::page_size);
 off_t offset_for_mmap =  align_down(offset, (off_t)mmu::page_size);

diff --git a/tests/tst-sendfile.cc b/tests/tst-sendfile.cc
--- a/tests/tst-sendfile.cc
+++ b/tests/tst-sendfile.cc
@@ -163,6 +163,22 @@ int test_sendfile_on_socket(off_t *offset, size_t  
count)

 return listener_result == 0 ? ret : -1;
 }

+int test_extents(int testfile_readfd, size_t offset, size_t count)
+{
+const char *out_file = "testdata_sendfile_output";
+off_t off;
+int write_fd = open(out_file, O_RDWR | O_TRUNC | O_CREAT, S_IRWXU);
+if (write_fd == -1) {
+printf("\topen() failed with error message  
= %s\n",strerror(errno));

+return -1;
+}
+lseek(testfile_readfd, 0 , SEEK_CUR);
+off = offset;
+int ret  = sendfile(write_fd, testfile_readfd, , count);
+close(write_fd);
+return ret;
+}
+
 int main()
 {
 int ret;
@@ -200,12 +216,22 @@ int main()
 report(test_functions[i](offset_p[j], count_array[k]) ==  
count_array[k], message.c_str());

 }
 offset = 0;
-   report(lseek(testfile_readfd, 0, SEEK_SET) == 0, "set readfd to  
beginning of file");
+report(lseek(testfile_readfd, 0, SEEK_SET) == 0, "set readfd  
to beginning of file");
 report(test_functions[i](offset_p[j], size_test_file) ==  
size_test_file, "copy entire file");

 printf("\n\n");
 }
 }

+// Test extents...
+report(lseek(testfile_readfd, 0, SEEK_SET) == 0, "set readfd to  
beginning of file");
+report(test_extents(testfile_readfd, 0, size_test_file + 1) ==  
size_test_file, "file size extent");
+report(lseek(testfile_readfd, 0, SEEK_SET) == 0, "set readfd to  
beginning of file");
+report(test_extents(testfile_readfd, 10, size_test_file) ==  
(size_test_file - 10), "file size extent with offset");
+report(lseek(testfile_readfd, 0, SEEK_SET) == 0, "set readfd to  
beginning of file");
+report(test_extents(testfile_readfd, size_test_file - 1, 1) ==  
1, "file size extent with offset (tail)");
+report(lseek(testfile_readfd, 0, SEEK_SET) == 0, "set readfd to  
beginning of file");
+report(test_extents(testfile_readfd, size_test_file, 1) == 0, "file  
size extent with offset (tail, no bytes)");

+
 /* force sendfile to fail in rest of test cases */
 ret = sendfile(100, testfile_readfd, NULL, 10);
 report(ret == -1 && errno == EBADF, "test for bad out_fd");

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] dhcp: do not release assigned IP just to update DNS name

[Commit Bot]

From: Justin Cinkelj 
Committer: Nadav Har'El 
Branch: master

dhcp: do not release assigned IP just to update DNS name

We want to propagate new hostname to DNS server via DHCP server.
Previously, a DHCP release followed by a new DORA cycle was used.
This did update DNS name, but VM was for a short period of time
without a valid IP.

Now an early lease renew (with the new hostname included) is sent.
Thus VM does not need to stop using assigned IP address.

Fixes #816

Signed-off-by: Justin Cinkelj 
Message-Id: <20161228221755.12758-1-justin.cink...@xlab.si>

---
diff --git a/core/dhcp.cc b/core/dhcp.cc
--- a/core/dhcp.cc
+++ b/core/dhcp.cc
@@ -72,10 +72,9 @@ void dhcp_release()
 net_dhcp_worker.release();
 }

-void dhcp_restart(bool wait)
+void dhcp_renew(bool wait)
 {
-net_dhcp_worker.release();
-net_dhcp_worker.start(wait);
+net_dhcp_worker.renew(wait);
 }

 namespace dhcp {
@@ -221,6 +220,9 @@ namespace dhcp {
 pkt->secs = 0;
 pkt->flags = 0;
 memcpy(pkt->chaddr, IF_LLADDR(ifp), ETHER_ADDR_LEN);
+ulong yip_n = htonl(yip.to_ulong());
+ulong sip_n = htonl(sip.to_ulong());
+memcpy(>ciaddr.s_addr, _n, 4);

 // Options
 u8* options_start = reinterpret_cast(pkt+1);
@@ -242,7 +244,7 @@ namespace dhcp {
 *options++ = DHCP_OPTION_END;

 dhcp_len += options - options_start;
-build_udp_ip_headers(dhcp_len, INADDR_ANY, INADDR_BROADCAST);
+build_udp_ip_headers(dhcp_len, yip_n, sip_n);
 }

 void dhcp_mbuf::compose_release(struct ifnet* ifp,
@@ -541,6 +543,22 @@ namespace dhcp {
 _client_addr = _server_addr = ipv4_zero;
 }

+void dhcp_interface_state::renew()
+{
+// Update state
+_state = DHCP_REQUEST;
+
+// Compose a dhcp request packet
+dhcp_mbuf dm(false);
+_xid = rand();
+dm.compose_request(_ifp,
+   _xid,
+   _client_addr, _server_addr);
+
+// Send
+_sock->dhcp_send(dm);
+}
+
 void dhcp_interface_state::process_packet(struct mbuf* m)
 {
 dhcp_mbuf dm(true, m);
@@ -692,13 +710,14 @@ namespace dhcp {
 _dhcp_thread->start();
 }

-void dhcp_worker::start(bool wait)
+void dhcp_worker::_send_and_wait(bool wait,  
dhcp_interface_state_send_packet iface_func)

 {
-// FIXME: clear routing table (use case run dhclient 2nd time)
+// When doing renew, we still have IP, but want to reuse the flag.
+_have_ip = false;
 do {
-// Send discover packets!
+// Send discover or renew packets!
 for (auto : _universe) {
-it.second->discover();
+(it.second->*iface_func)();
 }

 if (wait) {
@@ -714,6 +733,12 @@ namespace dhcp {
 } while (!_have_ip && wait);
 }

+void dhcp_worker::start(bool wait)
+{
+// FIXME: clear routing table (use case run dhclient 2nd time)
+_send_and_wait(wait, _interface_state::discover);
+}
+
 void dhcp_worker::release()
 {
 for (auto : _universe) {
@@ -724,6 +749,11 @@ namespace dhcp {
 usleep(1000);
 }

+void dhcp_worker::renew(bool wait)
+{
+_send_and_wait(wait, _interface_state::renew);
+}
+
 void dhcp_worker::dhcp_worker_fn()
 {
 while (true) {
diff --git a/include/osv/dhcp.hh b/include/osv/dhcp.hh
--- a/include/osv/dhcp.hh
+++ b/include/osv/dhcp.hh
@@ -25,7 +25,7 @@
 extern "C" {
 void dhcp_start(bool wait);
 void dhcp_release();
-void dhcp_restart(bool wait);
+void dhcp_renew(bool wait);
 }

 namespace dhcp {
@@ -226,6 +226,7 @@ namespace dhcp {

 void discover();
 void release();
+void renew();
 void process_packet(struct mbuf*);
 void state_discover(dhcp_mbuf );
 void state_request(dhcp_mbuf );
@@ -242,6 +243,8 @@ namespace dhcp {
 // Transaction id
 u32 _xid;
 };
+// typedef for discover/renew functions
+typedef void (dhcp_interface_state::*dhcp_interface_state_send_packet)  
(void);


  
///


@@ -256,6 +259,7 @@ namespace dhcp {
 void start(bool wait);
 // Send release packet for all DHCP IPs.
 void release();
+void renew(bool wait);

 void dhcp_worker_fn();
 void queue_packet(struct mbuf* m);
@@ -270,6 +274,7 @@ namespace dhcp {
 // Wait for IP
 bool _have_ip;
 sched::thread * _waiter;
+void _send_and_wait(bool wait, dhcp_interface_state_send_packet  
iface_func);

 };

 } // namespace dhcp
diff --git a/modules/cloud-init/cloud-init.cc  
b/modules/cloud-init/cloud-init.cc

--- a/modules/cloud-init/cloud-init.cc
+++ b/modules/cloud-init/cloud-init.cc
@@ -17,17 +17,17 @@
 #include 

 

[COMMIT osv master] align: add helper to check alignment

[Commit Bot]

From: Timmons C. Player 
Committer: Nadav Har'El 
Branch: master

align: add helper to check alignment

Signed-off-by: Timmons C. Player 
Message-Id: <1483547132-22868-2-git-send-email-timmons.pla...@spirent.com>

---
diff --git a/include/osv/align.hh b/include/osv/align.hh
--- a/include/osv/align.hh
+++ b/include/osv/align.hh
@@ -25,6 +25,13 @@ T align_up(T n, T alignment)
 return align_down(n + alignment - 1, alignment);
 }

+template 
+inline
+bool align_check(T n, T alignment)
+{
+return align_down(n, alignment) == n;
+}
+
 template 
 inline
 T* align_down(T* ptr, size_t alignment)
@@ -43,4 +50,11 @@ T* align_up(T* ptr, size_t alignment)
 return reinterpret_cast(n);
 }

+template 
+inline
+bool align_check(T* ptr, size_t alignment)
+{
+auto n = reinterpret_cast(ptr);
+return align_down(n, alignment) == n;
+}
 #endif

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] percpu: enforce alignment for dynamic percpu allocations

[Commit Bot]

From: Timmons C. Player 
Committer: Nadav Har'El 
Branch: master

percpu: enforce alignment for dynamic percpu allocations

If the dynamic memory buffer used for percpu allocations
wasn't congruent to the requested alignment, then the
returned memory wouldn't actually be aligned correctly.
This change forces the bitmap search to start at the proper
alignment.

Signed-off-by: Timmons C. Player 
Message-Id: <1483547132-22868-3-git-send-email-timmons.pla...@spirent.com>

---
diff --git a/core/percpu.cc b/core/percpu.cc
--- a/core/percpu.cc
+++ b/core/percpu.cc
@@ -8,12 +8,13 @@

 #include 
 #include 
+#include 
 #include 
+#include 

 static constexpr size_t dynamic_percpu_max = 65536;

-union dynamic_percpu_buffer {
-long align;
+struct dynamic_percpu_buffer {
 char buf[dynamic_percpu_max];
 };

@@ -28,8 +29,14 @@ size_t dynamic_percpu_base()

 size_t dynamic_percpu_alloc(size_t size, size_t align)
 {
+assert(is_power_of_two(align));
 std::lock_guard guard(mtx);
-for (size_t i = 0; i < dynamic_percpu_max; i += align) {
+
+/* Find the first value in the bitmap that has the necessary alignment  
*/

+auto percpu_base = dynamic_percpu_base();
+auto align_base = align_up(percpu_base, align) - percpu_base;
+
+for (size_t i = align_base; i < dynamic_percpu_max; i += align) {
 size_t j = 0;
 for (; j < size; ++j) {
 if (dynamic_percpu_allocated.test(i + j)) {
@@ -40,7 +47,8 @@ size_t dynamic_percpu_alloc(size_t size, size_t align)
 for (j = 0; j < size; ++j) {
 dynamic_percpu_allocated.set(i + j, true);
 }
-return dynamic_percpu_base() + i;
+assert(align_check(percpu_base + i, align));
+return percpu_base + i;
 }
 }
 abort("exhausted dynamic percpu pool");
@@ -54,4 +62,3 @@ void dynamic_percpu_free(size_t offset, size_t size)
 dynamic_percpu_allocated.set(offset + j, false);
 }
 }
-

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv-apps master] apps: Added openjdk8-zulu-compact3-with-java-beans and modified JDK java apps to allow building images in isolated or non-isolated mode

[Commit Bot]

From: Waldemar Kozaczuk 
Committer: Nadav Har'El 
Branch: master

apps: Added openjdk8-zulu-compact3-with-java-beans and modified JDK java  
apps to allow building images in isolated or non-isolated mode


Modified openjdk8-fedora ond openjdk9-ea-java-base to require  
new 'java-cmd' capability which can be
provided during build process by adding either java-isolated or  
java-non-isolated to the list of modules (see examples below).
The openjdk8-zulu-compactN apps on other hand implicitly include  
java-non-isolated module.


Added new openjdk8-ubuntu-profile3-with-java-beans app that is made of the  
compact3 profile plus java/beans and com/sun/beans
classes which makes is 1-2 MB bigger. This new app provides smallest JRE  
that can be used by Java apps that rely on

log4j 1.x, spring, etc. For example elasticsearch can run on this JRE.

Removed duplications in module.py and usr.manifest files.

Examples to build java apps:

scripts/build image=java,java-example
scripts/build image=java-non-isolated,openjdk8-fedora,java-example
scripts/build image=openjdk8-zulu-compact1,java-example

Signed-off-by: Waldemar Kozaczuk 

Message-Id: <1482156368-30213-1-git-send-email-jwkozac...@gmail.com>

---
diff --git a/.gitignore b/.gitignore
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,4 @@
 upstream
 src
 ROOTFS
+install
diff --git a/elasticsearch/Makefile b/elasticsearch/Makefile
--- a/elasticsearch/Makefile
+++ b/elasticsearch/Makefile
@@ -1,4 +1,4 @@
-VERSION:=2.4.0
+VERSION:=2.4.2
 NAME:=elasticsearch
 DIR:=$(NAME)-$(VERSION)
 TARBALL:=$(DIR).tar.gz
diff --git a/elasticsearch/module.py b/elasticsearch/module.py
--- a/elasticsearch/module.py
+++ b/elasticsearch/module.py
@@ -15,11 +15,10 @@
 '  -XX:+HeapDumpOnOutOfMemoryError '
 '  -XX:+DisableExplicitGC '
 '  -Dfile.encoding=UTF-8 '
-'  -Dosv.java.diagnostics '
 '  -Djna.nosys=true '
 '  -Des.insecure.allow.root=true '
 '  -Des.path.home=/elasticsearch '
 '  -Des.foreground=true '
-' -cp /elasticsearch/lib/elasticsearch-2.4.0.jar:/elasticsearch/lib/* '
+' -cp /elasticsearch/lib/elasticsearch-2.4.2.jar:/elasticsearch/lib/* '
 ' org.elasticsearch.bootstrap.Elasticsearch '
 ' start ')
diff --git a/elasticsearch/usr.manifest b/elasticsearch/usr.manifest
--- a/elasticsearch/usr.manifest
+++ b/elasticsearch/usr.manifest
@@ -0,0 +1 @@
+/elasticsearch/**: ${MODULE_DIR}/ROOTFS/elasticsearch/**
diff --git a/openjdk8-fedora-non-isolated/Makefile  
b/openjdk8-fedora-non-isolated/Makefile

--- a/openjdk8-fedora-non-isolated/Makefile
+++ b/openjdk8-fedora-non-isolated/Makefile
@@ -1,39 +0,0 @@
-#
-# Copyright (C) 2013 Cloudius Systems, Ltd.
-#
-# This work is open source software, licensed under the terms of the
-# BSD license as described in the LICENSE file in the top-level directory.
-#
-
-.PHONY: module clean
-
-jvm_dir=install/usr/lib/jvm
-
-java_version=$(shell ./latest.sh java-1.8.0-openjdk-headless x86_64)
-java_dir=http://mirrors.kernel.org/fedora/development/rawhide/Everything/x86_64/os/Packages/j/
-java_fullname=java-1.8.0-openjdk-headless-$(java_version).x86_64.rpm
-
-tzdata_version=$(shell ./latest.sh tzdata-java noarch)
-tzdata_dir=http://mirrors.kernel.org/fedora/development/rawhide/Everything/x86_64/os/Packages/t/
-tzdata_fullname=tzdata-java-$(tzdata_version).noarch.rpm
-
-SRC = $(shell readlink -f ../..)
-
-module:
-   cd $(SRC)/java && mvn package -q -DskipTests=true
-   mkdir -p upstream
-   wget -c -O upstream/$(java_fullname) $(java_dir)$(java_fullname)
-   wget -c -O upstream/$(tzdata_fullname) $(tzdata_dir)$(tzdata_fullname)
-   rm -rf install
-   mkdir -p install
-   rpm2cpio upstream/$(java_fullname) | (cd install && cpio -id)
-   ln -sf java-1.8.0-openjdk-$(java_version).x86_64 $(jvm_dir)/java
-   rm -rf $(jvm_dir)/java/jre/lib/security/cacerts
-   rm -rf $(jvm_dir)/java/jre/lib/audio/*
-   rpm2cpio upstream/$(tzdata_fullname) | (cd install && cpio -id)
-   ln -s /etc/pki/java/cacerts install/usr/lib/jvm/java/jre/lib/security/
-
-clean:
-   rm -rf upstream install
-   cd $(SRC)/java && mvn clean -q
-   -rm -f dependency-reduced-pom.xml
diff --git a/openjdk8-fedora-non-isolated/latest.sh  
b/openjdk8-fedora-non-isolated/latest.sh

--- a/openjdk8-fedora-non-isolated/latest.sh
+++ b/openjdk8-fedora-non-isolated/latest.sh
@@ -1,9 +0,0 @@
-#!/bin/bash
-
-package=$1
-
-arch=$2
-
-letter=${package:0:1}
-
-wget -qO-  
http://mirrors.kernel.org/fedora/development/rawhide/Everything/x86_64/os/Packages/$letter/  
| grep "$package-[0-9].*$arch" | sed -e "s/href\=\"$package-\(.*\)\.$arch\.rpm\".*/\1/g"
diff --git a/openjdk8-fedora-non-isolated/module.py  
b/openjdk8-fedora-non-isolated/module.py

--- a/openjdk8-fedora-non-isolated/module.py
+++ b/openjdk8-fedora-non-isolated/module.py
@@ -1,28 +0,0 @@
-#
-# Copyright (C) 2014 Cloudius Systems, Ltd.
-#
-# This work is open source software, licensed under the terms of the
-# BSD license as 

[COMMIT osv master] Add missing pthread_getname_np()

[Commit Bot]

From: Rick Payne 
Committer: Nadav Har'El 
Branch: master

Add missing pthread_getname_np()

Signed-off-by: Rick Payne 
Message-Id: <20161221123601.5879-1-ri...@rossfell.co.uk>

---
diff --git a/include/api/pthread.h b/include/api/pthread.h
--- a/include/api/pthread.h
+++ b/include/api/pthread.h
@@ -212,6 +212,7 @@ void _pthread_cleanup_pop(struct __ptcb *, int);
 #ifdef _GNU_SOURCE
 int pthread_getattr_np(pthread_t, pthread_attr_t *);
 int pthread_setname_np(pthread_t pthread, const char* name);
+int pthread_getname_np(pthread_t pthread, char* name, size_t namelen);
 int pthread_attr_setaffinity_np(pthread_attr_t *, size_t, const cpu_set_t  
*);
 int pthread_attr_getaffinity_np(const pthread_attr_t *, size_t, cpu_set_t  
*);

 int pthread_setaffinity_np(pthread_t, size_t, const cpu_set_t *);
diff --git a/libc/pthread.cc b/libc/pthread.cc
--- a/libc/pthread.cc
+++ b/libc/pthread.cc
@@ -968,6 +968,13 @@ int pthread_setname_np(pthread_t p, const char* name)
 return 0;
 }

+int pthread_getname_np(pthread_t p, char* name, size_t namelen)
+{
+  strncpy(name, pthread::from_libc(p)->_thread->name().c_str(), namelen);
+  name[namelen-1] = 0;
+  return 0;
+}
+
 int pthread_attr_setaffinity_np(pthread_attr_t *attr, size_t cpusetsize,
 const cpu_set_t *cpuset)
 {

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] Move vfprintf.c from musl/ to libc/

[Commit Bot]

From: Nadav Har'El 
Committer: Nadav Har'El 
Branch: master

Move vfprintf.c from musl/ to libc/

Copy vfprintf.c from musl/ to libc/, unchanged, in preperation for
fixing the bug in issue #818.

Signed-off-by: Nadav Har'El 
Message-Id: <20161214163437.14901-1-...@scylladb.com>

---
diff --git a/Makefile b/Makefile
--- a/Makefile
+++ b/Makefile
@@ -1490,7 +1490,7 @@ musl += stdio/ungetc.o
 musl += stdio/ungetwc.o
 musl += stdio/vasprintf.o
 libc += stdio/vdprintf.o
-musl += stdio/vfprintf.o
+libc += stdio/vfprintf.o
 libc += stdio/vfscanf.o
 musl += stdio/vfwprintf.o
 libc += stdio/vfwscanf.o
diff --git a/libc/stdio/vfprintf.c b/libc/stdio/vfprintf.c
--- a/libc/stdio/vfprintf.c
+++ b/libc/stdio/vfprintf.c
@@ -0,0 +1,677 @@
+#include "stdio_impl.h"
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+/* Some useful macros */
+
+#define MAX(a,b) ((a)>(b) ? (a) : (b))
+#define MIN(a,b) ((a)<(b) ? (a) : (b))
+#define CONCAT2(x,y) x ## y
+#define CONCAT(x,y) CONCAT2(x,y)
+
+/* Convenient bit representation for modifier flags, which all fall
+ * within 31 codepoints of the space character. */
+
+#define ALT_FORM   (1U<<'#'-' ')
+#define ZERO_PAD   (1U<<'0'-' ')
+#define LEFT_ADJ   (1U<<'-'-' ')
+#define PAD_POS(1U<<' '-' ')
+#define MARK_POS   (1U<<'+'-' ')
+#define GROUPED(1U<<'\''-' ')
+
+#define FLAGMASK (ALT_FORM|ZERO_PAD|LEFT_ADJ|PAD_POS|MARK_POS|GROUPED)
+
+#if UINT_MAX == ULONG_MAX
+#define LONG_IS_INT
+#endif
+
+#if SIZE_MAX != ULONG_MAX || UINTMAX_MAX != ULLONG_MAX
+#define ODD_TYPES
+#endif
+
+/* State machine to accept length modifiers + conversion specifiers.
+ * Result is 0 on failure, or an argument type to pop on success. */
+
+enum {
+   BARE, LPRE, LLPRE, HPRE, HHPRE, BIGLPRE,
+   ZTPRE, JPRE,
+   STOP,
+   PTR, INT, UINT, ULLONG,
+#ifndef LONG_IS_INT
+   LONG, ULONG,
+#else
+#define LONG INT
+#define ULONG UINT
+#endif
+   SHORT, USHORT, CHAR, UCHAR,
+#ifdef ODD_TYPES
+   LLONG, SIZET, IMAX, UMAX, PDIFF, UIPTR,
+#else
+#define LLONG ULLONG
+#define SIZET ULONG
+#define IMAX LLONG
+#define UMAX ULLONG
+#define PDIFF LONG
+#define UIPTR ULONG
+#endif
+   DBL, LDBL,
+   NOARG,
+   MAXSTATE
+};
+
+#define S(x) [(x)-'A']
+
+static const unsigned char states[]['z'-'A'+1] = {
+   { /* 0: bare types */
+   S('d') = INT, S('i') = INT,
+   S('o') = UINT, S('u') = UINT, S('x') = UINT, S('X') = UINT,
+   S('e') = DBL, S('f') = DBL, S('g') = DBL, S('a') = DBL,
+   S('E') = DBL, S('F') = DBL, S('G') = DBL, S('A') = DBL,
+   S('c') = CHAR, S('C') = INT,
+   S('s') = PTR, S('S') = PTR, S('p') = UIPTR, S('n') = PTR,
+   S('m') = NOARG,
+   S('l') = LPRE, S('h') = HPRE, S('L') = BIGLPRE,
+   S('z') = ZTPRE, S('j') = JPRE, S('t') = ZTPRE,
+   }, { /* 1: l-prefixed */
+   S('d') = LONG, S('i') = LONG,
+   S('o') = ULONG, S('u') = ULONG, S('x') = ULONG, S('X') = ULONG,
+   S('e') = DBL, S('f') = DBL, S('g') = DBL, S('a') = DBL,
+   S('E') = DBL, S('F') = DBL, S('G') = DBL, S('A') = DBL,
+   S('c') = INT, S('s') = PTR, S('n') = PTR,
+   S('l') = LLPRE,
+   }, { /* 2: ll-prefixed */
+   S('d') = LLONG, S('i') = LLONG,
+   S('o') = ULLONG, S('u') = ULLONG,
+   S('x') = ULLONG, S('X') = ULLONG,
+   S('n') = PTR,
+   }, { /* 3: h-prefixed */
+   S('d') = SHORT, S('i') = SHORT,
+   S('o') = USHORT, S('u') = USHORT,
+   S('x') = USHORT, S('X') = USHORT,
+   S('n') = PTR,
+   S('h') = HHPRE,
+   }, { /* 4: hh-prefixed */
+   S('d') = CHAR, S('i') = CHAR,
+   S('o') = UCHAR, S('u') = UCHAR,
+   S('x') = UCHAR, S('X') = UCHAR,
+   S('n') = PTR,
+   }, { /* 5: L-prefixed */
+   S('e') = LDBL, S('f') = LDBL, S('g') = LDBL, S('a') = LDBL,
+   S('E') = LDBL, S('F') = LDBL, S('G') = LDBL, S('A') = LDBL,
+   S('n') = PTR,
+   }, { /* 6: z- or t-prefixed (assumed to be same size) */
+   S('d') = PDIFF, S('i') = PDIFF,
+   S('o') = SIZET, S('u') = SIZET,
+   S('x') = SIZET, S('X') = SIZET,
+   S('n') = PTR,
+   }, { /* 7: j-prefixed */
+   S('d') = IMAX, S('i') = IMAX,
+   S('o') = UMAX, S('u') = UMAX,
+   S('x') = UMAX, S('X') = UMAX,
+   S('n') = PTR,
+   }
+};
+
+#define OOB(x) ((unsigned)(x)-'A' > 'z'-'A')
+
+union arg
+{
+   uintmax_t i;
+   long double f;
+   void *p;
+};
+
+static void pop_arg(union arg *arg, int type, va_list *ap)
+{
+   /* Give the compiler a hint for optimizing the switch. */
+   if ((unsigned)type > MAXSTATE) return;
+   switch (type) 

[COMMIT osv master] printf(): fix "L" and "ll" prefixes

[Commit Bot]

From: Nadav Har'El 
Committer: Nadav Har'El 
Branch: master

printf(): fix "L" and "ll" prefixes

Musl, like Posix, allows the "L" prefix to be used only on long doubles,
and "ll" prefix on long long int. However, glibc actually allows both on
both types.

So this patch changes both prefixes to allow both types, to work like Linux.
It also adds a test that this works.

We saw this problem in an actual application, which used the "%Lx" format,
that worked on Linux but not on OSv (on OSv, the entire printf did not
happen because of the format parse error).

Fixes #818.

Signed-off-by: Nadav Har'El 
Message-Id: <20161214163437.14901-2-...@scylladb.com>

---
diff --git a/libc/stdio/vfprintf.c b/libc/stdio/vfprintf.c
--- a/libc/stdio/vfprintf.c
+++ b/libc/stdio/vfprintf.c
@@ -91,6 +91,10 @@ static const unsigned char states[]['z'-'A'+1] = {
S('o') = ULLONG, S('u') = ULLONG,
S('x') = ULLONG, S('X') = ULLONG,
S('n') = PTR,
+   // unlike Posix, glibc also allows ll prefix for long double,
+   // same as L prefix:
+   S('e') = LDBL, S('f') = LDBL, S('g') = LDBL, S('a') = LDBL,
+   S('E') = LDBL, S('F') = LDBL, S('G') = LDBL, S('A') = LDBL,
}, { /* 3: h-prefixed */
S('d') = SHORT, S('i') = SHORT,
S('o') = USHORT, S('u') = USHORT,
@@ -106,6 +110,11 @@ static const unsigned char states[]['z'-'A'+1] = {
S('e') = LDBL, S('f') = LDBL, S('g') = LDBL, S('a') = LDBL,
S('E') = LDBL, S('F') = LDBL, S('G') = LDBL, S('A') = LDBL,
S('n') = PTR,
+   // unlike Posix, glibc also allows L prefix for long long int,
+   // same as ll prefix:
+   S('d') = LLONG, S('i') = LLONG,
+   S('o') = ULLONG, S('u') = ULLONG,
+   S('x') = ULLONG, S('X') = ULLONG,
}, { /* 6: z- or t-prefixed (assumed to be same size) */
S('d') = PDIFF, S('i') = PDIFF,
S('o') = SIZET, S('u') = SIZET,
diff --git a/tests/tst-printf.cc b/tests/tst-printf.cc
--- a/tests/tst-printf.cc
+++ b/tests/tst-printf.cc
@@ -15,6 +15,7 @@

 #include 
 #include 
+#include 

 static int tests = 0, fails = 0;

@@ -30,6 +31,27 @@ static void report(bool ok, std::string msg)
 // some builtin. So we need to use this variable
 const char *format = "%s";

+
+std::string print(const char* fmt, ...){
+int size = 512;
+char* buffer = 0;
+buffer = new char[size];
+va_list vl;
+va_start(vl, fmt);
+int nsize = vsnprintf(buffer, size, fmt, vl);
+if(size<=nsize){ //fail delete buffer and try again
+delete[] buffer;
+buffer = 0;
+buffer = new char[nsize+1]; //+1 for /0
+nsize = vsnprintf(buffer, size, fmt, vl);
+}
+std::string ret(buffer);
+va_end(vl);
+delete[] buffer;
+return ret;
+}
+
+
 int main(int ac, char** av)
 {
 // Test that when snprintf is given a >32-bit length, including -1,
@@ -44,5 +66,17 @@ int main(int ac, char** av)
 report(snprintf(dest2, 1ULL<<40, format, source) == 5, "snprintf with  
n=2^40");

 report(strcmp(source, dest2) == 0, "strcmp that result");

+// Posix states that the "L" prefix is for long double, and "ll" is
+// for long long int, but in Linux glibc these are actually synonyms,
+// and either can be used
+// Test that the L format prefix, for "long double", works
+long double d = 123.456;
+long long int i = 123456;
+report(print("%Lf", d) == "123.456000", "Lf");
+report(print("%llf", d) == "123.456000", "llf");
+report(print("%Ld", i) == "123456", "Ld");
+report(print("%lld", i) == "123456", "lld");
+
 std::cout << "SUMMARY: " << tests << " tests, " << fails << "  
failures\n";

+return (fails != 0);
 }

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] osv::run const correctness

[Commit Bot]

From: Nadav Har'El 
Committer: Nadav Har'El 
Branch: master

osv::run const correctness

The variant of osv::run which takes argv does not need the strings or
their array to be writable, since it copies them into an std::vector
anyway, so we can take a const char* const* instead of a char**.

Also fix non-const string warnings in tst-run.cc.

Signed-off-by: Nadav Har'El 

---
diff --git a/core/run.cc b/core/run.cc
--- a/core/run.cc
+++ b/core/run.cc
@@ -16,7 +16,7 @@ std::shared_ptr run(std::string path,
 }

 std::shared_ptr run(std::string path,
- int argc, char** argv, int *return_code)
+ int argc, const char* const* argv, int  
*return_code)

 {
 std::vector args;
 for (int i = 0; i < argc; i++) {
diff --git a/include/osv/run.hh b/include/osv/run.hh
--- a/include/osv/run.hh
+++ b/include/osv/run.hh
@@ -78,7 +78,7 @@ namespace osv {
  * \return \c shared pointer to the application
  */
 std::shared_ptr run(std::string path,
-  int argc, char** argv, int  
*return_code);
+  int argc, const char* const* argv,  
int *return_code);


 /**
  * Run the given executable.
diff --git a/tests/tst-run.cc b/tests/tst-run.cc
--- a/tests/tst-run.cc
+++ b/tests/tst-run.cc
@@ -31,7 +31,7 @@ int main(int ac, char** av)
 }

 // See that I can run myself (with a special argument to stop the  
recursion)

-char *child_args[] = {"/tests/tst-run.so", ""};
+const char *child_args[] = {"/tests/tst-run.so", ""};
 int ret;
 bool b = (bool)osv::run("/tests/tst-run.so", 2, child_args, );
 report(b == true, "Run myself");

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] libc: move pthread_barrier to a separate file

[Commit Bot]

From: Justin Cinkelj 
Committer: Nadav Har'El 
Branch: master

libc: move pthread_barrier to a separate file

As @nyh noted in #823 gcc 4.8 doesn't like both using and redefining
pthread_mutex_lock in the same file. This is a workaround for what
seems to be a compiler bug. The code using pthread_mutex_lock/unlock
(pthread_barrier_* functions) is moved to a new .cc file.

Fixes #823

Signed-off-by: Justin Cinkelj 
Message-Id: <20161228201949.7322-1-justin.cink...@xlab.si>

---
diff --git a/Makefile b/Makefile
--- a/Makefile
+++ b/Makefile
@@ -1652,6 +1652,7 @@ musl += regex/tre-mem.o
 $(out)/musl/src/regex/tre-mem.o: CFLAGS += -UNDEBUG

 libc += pthread.o
+libc += pthread_barrier.o
 libc += libc.o
 libc += dlfcn.o
 libc += time.o
diff --git a/libc/pthread.cc b/libc/pthread.cc
--- a/libc/pthread.cc
+++ b/libc/pthread.cc
@@ -29,7 +29,7 @@

 #include 
 #include 
-#include 
+
 #include "pthread.hh"

 namespace pthread_private {
@@ -1128,117 +1128,3 @@ int pthread_attr_getaffinity_np(const  
pthread_attr_t *attr, size_t cpusetsize,


 return 0;
 }
-
-// Private definitions of the internal structs backing pthread_barrier_t  
and

-// pthread_barrierattr_t
-typedef struct
-{
-unsigned int out;
-unsigned int count;
-latch *ltch;
-pthread_mutex_t *mtx;
-} pthread_barrier_t_int;
-
-typedef struct
-{
-unsigned pshared;
-} pthread_barrierattr_t_int;
-
-int pthread_barrier_init(pthread_barrier_t *barrier_opq,
- const pthread_barrierattr_t *attr_opq,
- unsigned count)
-{
-pthread_barrier_t_int *barrier = (pthread_barrier_t_int*) barrier_opq;
-static_assert(sizeof(pthread_barrier_t_int) <=  
sizeof(pthread_barrier_t),
-  "pthread_barrier_t_int is larger than  
pthread_barrier_t");

-
-// Linux returns EINVAL if count == 0 or INT_MAX so we do too.
-// In theory, we could go up to UINT_MAX since count is unsigned.
-if (!barrier || count == 0 || count >= INT_MAX) {
-return EINVAL;
-}
-
-// Always ignore attr, it has no meaning in the context of a unikernel.
-// pthread_barrierattr_t has a single member variable pshared that can  
be set

-// to PTHREAD_PROCESS_PRIVATE or PTHREAD_PROCESS_SHARED. These have the
-// same effect in a unikernel - there is only a single process and all
-// threads can manipulate the memory area associated with the
-// pthread_barrier_t so it doesn't matter what the value of pshared is  
set to

-barrier->count = count;
-barrier->out = 0;
-barrier->ltch = new latch(count);
-barrier->mtx = new pthread_mutex_t;
-pthread_mutex_init(barrier->mtx, NULL);
-return 0;
-}
-
-int pthread_barrier_wait(pthread_barrier_t *barrier_opq)
-{
-pthread_barrier_t_int *barrier = (pthread_barrier_t_int*) barrier_opq;
-static_assert(sizeof(pthread_barrier_t_int) <=  
sizeof(pthread_barrier_t),
-  "pthread_barrier_t_int is larger than  
pthread_barrier_t");

-
-if (!barrier || !barrier->ltch || !barrier->mtx) {
-return EINVAL;
-}
-
-int retval = 0;
-pthread_mutex_t *mtx = barrier->mtx;
-
-pthread_mutex_lock(mtx);
-pthread_mutex_unlock(mtx);
-
-latch *l  = barrier->ltch;
-l->count_down();
-// All threads stuck here until we get at least 'count' waiters
-l->await();
-
-// If the last thread (thread x) to wait on the barrier is descheduled  
here

-// (immediately after being the count'th thread crossing the barrier)
-// the barrier remains open (a new waiting thread will cross) until
-// the barrier is reset below (when thread x is rescheduled), which  
doesn't
-// seem technically incorrect. Only one of the crossing threads will  
get a

-// retval of PTHREAD_BARRIER_SERIAL_THREAD, when
-// barrier->out == barrier->count.
-// All other crossing threads will get a retval of 0.
-
-pthread_mutex_lock(mtx);
-barrier->out++;
-// Make the last thread out responsible for resetting the barrier's  
latch.

-// The last thread also gets the special return value
-// PTHREAD_BARRIER_SERIAL_THREAD. Every other thread gets a retval of 0
-if (barrier->out == barrier->count) {
-retval = PTHREAD_BARRIER_SERIAL_THREAD;
-// Reset the latch for the next round of waiters. We're using an
-// external lock (mtx) to ensure that no other thread is calling
-// count_down or in await when we're resetting it. Without the  
external

-// lock, resetting the latch isn't safe.
-l->unsafe_reset(barrier->count);
-// Reset the 'out' counter so that the equality check above works  
across

-// multiple rounds of threads waiting on the barrier
-barrier->out = 0;
-}
-pthread_mutex_unlock(mtx);
-return retval;
-}
-
-int pthread_barrier_destroy(pthread_barrier_t *barrier_opq)
-{
-pthread_barrier_t_int *barrier = 

[COMMIT osv master] build: strip binaries in bootfs as well

[Commit Bot]

From: Nadav Har'El 
Committer: Nadav Har'El 
Branch: master

build: strip binaries in bootfs as well

Before this patch, mkbootfs.py, unlike upload_manifest.py, did not do
stripping. This was leading to unnecessary inflating of the bootfs.
With this patch, shared objects copied to the bootfs are also stripped.

As an example, the image created by "scripts/build image=httpserver"
goes down from 17 MB to 16 MB.

Fixes #667.
Refs #820.

Signed-off-by: Yuri Volchkov 
Signed-off-by: Nadav Har'El 

---
diff --git a/scripts/mkbootfs.py b/scripts/mkbootfs.py
--- a/scripts/mkbootfs.py
+++ b/scripts/mkbootfs.py
@@ -1,6 +1,6 @@
 #!/usr/bin/python

-import os, struct, optparse, io
+import os, struct, optparse, io, subprocess
 try:
 import configparser
 except ImportError:
@@ -52,6 +52,23 @@ def read_manifest(fn):
   for f in manifest.options('manifest')])
 return files

+def to_strip(filename):
+ff = os.path.abspath(filename)
+osvdir = os.path.abspath('../..')
+return ff.startswith(os.getcwd()) or \
+ff.startswith(osvdir + "/modules") or \
+ff.startswith(osvdir + "/apps")
+
+def strip_file(filename):
+stripped_filename = filename
+if filename.endswith(".so") and to_strip(filename):
+stripped_filename = filename[:-3] + "-stripped.so"
+if not os.path.exists(stripped_filename) \
+or (os.path.getmtime(stripped_filename) < \
+os.path.getmtime(filename)):
+subprocess.call(["strip", "-o", stripped_filename, filename])
+return stripped_filename
+
 def main():
 make_option = optparse.make_option

@@ -90,6 +107,8 @@ def main():
 files = read_manifest(options.manifest)
 files = list(expand(files.items()))
 files = [(x, unsymlink(y)) for (x, y) in files]
+files = [(x, y) for (x, y) in files if not x.endswith("-stripped.so")]
+files = [(x, strip_file(y)) for (x, y) in files]

 pos = (len(files) + 1) * metadata_size

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] build: strip binaries in modules/ dir as well

[Commit Bot]

From: Nadav Har'El 
Committer: Nadav Har'El 
Branch: master

build: strip binaries in modules/ dir as well

upload_manifest.py stripped only binaries in the build/ directory, so
binaries in modules, such as modules/httpserver/libhttpserver.so were
not stripped.

Arguably, each module's Makefile could strip its own results, but it's
convenient to keep both the unstripped binary (for debugging) and stripped
one (for the image). So this patch extends to modules/ and apps/ the
already existing mechanism which when uploading a file something.so,
strips it into something-stripped.so and uploads that.

The patch also ensures that even if something-stripped.so matches a
wildcard it is not independently upload to the image (causing a loop).

With this patch, the image generated by "scripts/build image=httpserver"
is reduced from 25 MB to 17 MB.

Refs #822

Signed-off-by: Yuri Volchkov 
Signed-off-by: Nadav Har'El 

---
diff --git a/scripts/upload_manifest.py b/scripts/upload_manifest.py
--- a/scripts/upload_manifest.py
+++ b/scripts/upload_manifest.py
@@ -118,10 +118,16 @@ def cpio_header(filename, mode, filesize):
 + cpio_field(0, 8)# check
 + filename + b'\0')

+def to_strip(filename):
+ff = os.path.abspath(filename);
+osvdir = os.path.abspath('../..');
+return ff.startswith(os.getcwd()) or \
+ff.startswith(osvdir + "/modules") or \
+ff.startswith(osvdir + "/apps")
+
 def strip_file(filename):
 stripped_filename = filename
-if filename.endswith(".so") and \
-(filename[0] != "/" or filename.startswith(os.getcwd())):
+if filename.endswith(".so") and to_strip(filename):
 stripped_filename = filename[:-3] + "-stripped.so"
 if not os.path.exists(stripped_filename) \
 or (os.path.getmtime(stripped_filename) < \
@@ -138,6 +144,8 @@ def strip_file(filename):
 cpio_send(link.encode())
 else:
 depends.write('\t%s \\\n' % (hostname,))
+if hostname.endswith("-stripped.so"):
+continue
 hostname = strip_file(hostname)
 if os.path.islink(hostname):
 perm = os.lstat(hostname).st_mode & 0o777

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] build: possibly use cross-compilation strip

[Commit Bot]

From: Nadav Har'El 
Committer: Nadav Har'El 
Branch: master

build: possibly use cross-compilation strip

In the previous patch, scripts/mkbootfs.py started to use 'strip'.
However, this is run from the Makefile which allows cross-compilation,
and in that case, we need to use $STRIP instead of 'strip'.

So in this patch, we export the correct $STRIP from the Makefile to
its children, and use it in scripts/mkbootfs.py.

Signed-off-by: Nadav Har'El 

---
diff --git a/Makefile b/Makefile
--- a/Makefile
+++ b/Makefile
@@ -88,7 +88,7 @@ CROSS_PREFIX ?= $(if $(filter-out $(arch), $(host_arch)),  
$(arch)-linux-gnu-)

 CXX=$(CROSS_PREFIX)g++
 CC=$(CROSS_PREFIX)gcc
 LD=$(CROSS_PREFIX)ld.bfd
-STRIP=$(CROSS_PREFIX)strip
+export STRIP=$(CROSS_PREFIX)strip
 OBJCOPY=$(CROSS_PREFIX)objcopy

 # Our makefile puts all compilation results in a single directory, $(out),
diff --git a/scripts/mkbootfs.py b/scripts/mkbootfs.py
--- a/scripts/mkbootfs.py
+++ b/scripts/mkbootfs.py
@@ -66,7 +66,7 @@ def strip_file(filename):
 if not os.path.exists(stripped_filename) \
 or (os.path.getmtime(stripped_filename) < \
 os.path.getmtime(filename)):
-subprocess.call(["strip", "-o", stripped_filename, filename])
+subprocess.call([os.getenv("STRIP", "strip"), "-o",  
stripped_filename, filename])

 return stripped_filename

 def main():

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] loader: do not kill httpserver prematurely

[Commit Bot]

From: Nadav Har'El 
Committer: Nadav Har'El 
Branch: master

loader: do not kill httpserver prematurely

Commit e270cf9c (more than two years ago) was well-intentioned, but
introduced a new bug: we no longer wait for the background programs
to finish before stopping httpserver. So for example if the command
line is something like

   /cli/cli.so & finish_quickly.so

And finish_quickly.so finishes quickly, we do not wait for cli.so to
finish before killing httpserver.

One user discovered that because of that, the image created by
"./scripts/build image=java-example,cli" runs the short-lived java-example
but then the CLI doesn't work: It can't run any commands because the
httpserver it relies on is already dead.

So this patch partially reverts e270cf9c... We keep the "app registry"
introduced there, but we still have a separate list of background apps
started in the command line - and wait for them to finish before asking
httpserver to exit.

Signed-off-by: Nadav Har'El 
Message-Id: <20161225220455.8746-1-...@scylladb.com>

---
diff --git a/loader.cc b/loader.cc
--- a/loader.cc
+++ b/loader.cc
@@ -495,6 +495,7 @@ void* do_main_thread(void *_main_args)
 // empty otherwise, to run in this thread. '&!' is the same as '&', but
 // doesn't wait for the thread to finish before exiting OSv.
 std::vector detached;
+std::vector bg;
 for (auto  : commands) {
 std::vector newvec(it.begin(), std::prev(it.end()));
 auto suffix = it.back();
@@ -505,13 +506,19 @@ void* do_main_thread(void *_main_args)
 detached.push_back(app);
 } else if (!background) {
 app->join();
+} else {
+bg.push_back(app);
 }
 } catch (const launch_error& e) {
 std::cerr << e.what() << ". Powering off.\n";
 osv::poweroff();
 }
 }

+for (auto app : bg) {
+app->join();
+}
+
 for (auto app : detached) {
 app->request_termination();
 debug("Requested termination of %s, waiting...\n",  
app->get_command());


--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] aarch64: fix compilation on gcc 6

[Commit Bot]

From: Nadav Har'El 
Committer: Nadav Har'El 
Branch: master

aarch64: fix compilation on gcc 6

Gcc 6 doesn't like the member "x" being redefined, and starts
complaining about its visibility. If it's just a shortcut we want,
let's just use auto& and not a member. This revives "make ARCH=aarch64"
on gcc 6.2.1.

Signed-off-by: Nadav Har'El 

---
diff --git a/arch/aarch64/arch-mmu.hh b/arch/aarch64/arch-mmu.hh
--- a/arch/aarch64/arch-mmu.hh
+++ b/arch/aarch64/arch-mmu.hh
@@ -37,19 +37,19 @@ public:

 /* false->non-shareable true->Inner Shareable */
 inline void set_share(bool v) {
+auto& x=pt_element_common::x;
 x &= ~(3ul << 8);
 if (v)
 x |= (3ul << 8);
 }

 // mair_el1 register defines values for each 8 indexes. See boot.S
 inline void set_attridx(unsigned char c) {
+auto& x=pt_element_common::x;
 assert(c <= 7);
 x &= ~(7ul << 2);
 x |= (c << 2);
 }
-private:
-using pt_element_common::x;
 };


--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] makefile: remove space from cross-compilation prefix

[Commit Bot]

From: Nadav Har'El 
Committer: Nadav Har'El 
Branch: master

makefile: remove space from cross-compilation prefix

The spaces in $(if ...) are not ignored - they result in a space
in the beginning of the CROSS_PREFIX variable. This is mostly harmless,
except that mkbootfs.py now uses $STRIP and because it passes it to
popen, not the shell, it doesn't like that it begins with a space.

Signed-off-by: Nadav Har'El 

---
diff --git a/Makefile b/Makefile
--- a/Makefile
+++ b/Makefile
@@ -84,7 +84,7 @@ ifeq (,$(wildcard conf/$(arch).mk))
 endif
 include conf/$(arch).mk

-CROSS_PREFIX ?= $(if $(filter-out $(arch), $(host_arch)),  
$(arch)-linux-gnu-)

+CROSS_PREFIX ?= $(if $(filter-out $(arch),$(host_arch)),$(arch)-linux-gnu-)
 CXX=$(CROSS_PREFIX)g++
 CC=$(CROSS_PREFIX)gcc
 LD=$(CROSS_PREFIX)ld.bfd

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] DHCP - additional refinements

[Commit Bot]

From: Justin Cinkelj 
Committer: Nadav Har'El 
Branch: master

DHCP - additional refinements

From RFC 2131, page 30-31
o DHCPREQUEST generated during SELECTING state:
  Client inserts the address of the selected server in 'server
  identifier', 'ciaddr' MUST be zero, 'requested IP address MUST be
  filled in with the yiaddr value from the chosen DHCPOFFER.

o DHCPREQUEST generated during RENEWING state:
  'server identifier' MUST NOT be filled in, 'requested IP address'
  option MUST NOT be filled in, 'ciaddr' MUST be filled in with
  client's IP address.

I understand "'server identifier'/'requested IP address' MUST NOT be filled  
in"

as instruction to not include those two DHCP OPTIONS into RENEWING/REBINDING
DHCPREQUEST. So I tried to add additional change, as an attempt to be more  
RFC
compliant. After that change, code still works - well, at least against  
dnsmasq.


Waldek Kozaczuk tested against InfoBlox and in AWS and it works.

Message-Id:  
<1436431666.124738710.1490222473623.javamail.zim...@zimbra.xlab.si>


---
diff --git a/core/dhcp.cc b/core/dhcp.cc
--- a/core/dhcp.cc
+++ b/core/dhcp.cc
@@ -253,12 +253,16 @@ namespace dhcp {
 ip::address_v4::bytes_type dhcp_server_ip = sip.to_bytes();
 ip::address_v4::bytes_type requested_ip = yip.to_bytes();
 options = add_option(options, DHCP_OPTION_MESSAGE_TYPE, 1,  
DHCP_MT_REQUEST);
-options = add_option(options, DHCP_OPTION_DHCP_SERVER, 4,  
(u8*)_server_ip);

+if(request_packet_type == DHCP_REQUEST_SELECTING) {
+options = add_option(options, DHCP_OPTION_DHCP_SERVER, 4,  
(u8*)_server_ip);

+}
 char hostname[256];
 if (0 == gethostname(hostname, sizeof(hostname))) {
 options = add_option(options, DHCP_OPTION_HOSTNAME,  
strlen(hostname), (u8*)hostname);

 }
-options = add_option(options, DHCP_OPTION_REQUESTED_ADDRESS, 4,  
(u8*)_ip);

+if(request_packet_type == DHCP_REQUEST_SELECTING) {
+options = add_option(options, DHCP_OPTION_REQUESTED_ADDRESS,  
4, (u8*)_ip);

+}
 options = add_option(options, DHCP_OPTION_PARAMETER_REQUEST_LIST,
 sizeof(requested_options), requested_options);
 *options++ = DHCP_OPTION_END;

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] Changed DHCP client logic to properly broadcast DHCPREQUEST during

[Commit Bot]

From: wkozaczuk 
Committer: Nadav Har'El 
Branch: master

Changed DHCP client logic to properly broadcast DHCPREQUEST during

The patch addressing issue #816 changed dhcp_mbuf::compose_request() method  
to
make it unicast DHCPREQUEST message to a DHCP server. This is incorrect per  
RCF-2131
which requires DHCP clients to broadcast messages until IP address is bound  
which
is a case during RENEWING phase but not SELECTING one. Now the  
dhcp_mbuf::compose_request()
takes extra enum parameter which specifies which phase (per page 34 of  
https://www.ietf.org/rfc/rfc2131.txt)

the message is being composed for.

Fixes #864

Signed-off-by: Waldemar Kozaczuk 
Message-Id: <1490187468-6838-1-git-send-email-jwkozac...@gmail.com>

---
diff --git a/core/dhcp.cc b/core/dhcp.cc
--- a/core/dhcp.cc
+++ b/core/dhcp.cc
@@ -81,11 +81,26 @@ namespace dhcp {

 const char * dhcp_options_magic = "\x63\x82\x53\x63";

+static std::map  
dhcp_message_type_name_by_id = {

+{DHCP_MT_DISCOVER, "DHCPDISCOVER"},
+{DHCP_MT_OFFER, "DHCPOFFER"},
+{DHCP_MT_REQUEST, "DHCPREQUEST"},
+{DHCP_MT_DECLINE, "DHCPDECLINE"},
+{DHCP_MT_ACK, "DHCPACK"},
+{DHCP_MT_NAK, "DHCPNAK"},
+{DHCP_MT_RELEASE, "DHCPRELEASE"},
+{DHCP_MT_INFORM, "DHCPINFORM"},
+{DHCP_MT_LEASEQUERY, "DHCPLEASEQUERY"},
+{DHCP_MT_LEASEUNASSIGNED, "DHCPLEASEUNASSIGNED"},
+{DHCP_MT_LEASEUNKNOWN, "DHCPLEASEUNKNOWN"},
+{DHCP_MT_LEASEACTIVE, "DHCPLEASEACTIVE"},
+{DHCP_MT_INVALID, "DHCPINVALID"}
+};
+
  
///


 bool dhcp_socket::dhcp_send(dhcp_mbuf& packet)
 {
-
 struct bsd_sockaddr dst = {};
 struct mbuf *m = packet.get();

@@ -205,7 +220,8 @@ namespace dhcp {
 void dhcp_mbuf::compose_request(struct ifnet* ifp,
 u32 xid,
 ip::address_v4 yip,
-ip::address_v4 sip)
+ip::address_v4 sip,
+dhcp_request_packet_type  
request_packet_type)

 {
 size_t dhcp_len = sizeof(struct dhcp_packet);
 struct dhcp_packet* pkt = pdhcp();
@@ -222,7 +238,11 @@ namespace dhcp {
 memcpy(pkt->chaddr, IF_LLADDR(ifp), ETHER_ADDR_LEN);
 ulong yip_n = htonl(yip.to_ulong());
 ulong sip_n = htonl(sip.to_ulong());
-memcpy(>ciaddr.s_addr, _n, 4);
+if(request_packet_type == DHCP_REQUEST_RENEWING ||  
request_packet_type == DHCP_REQUEST_REBINDING) {

+// ciaddr should only be set to if RENEWING or REBINDING
+// See pages 21 and 30-31 in  
https://www.ietf.org/rfc/rfc2131.txt

+memcpy(>ciaddr.s_addr, _n, 4);
+}

 // Options
 u8* options_start = reinterpret_cast(pkt+1);
@@ -244,7 +264,14 @@ namespace dhcp {
 *options++ = DHCP_OPTION_END;

 dhcp_len += options - options_start;
-build_udp_ip_headers(dhcp_len, yip_n, sip_n);
+
+// See page 33 in https://www.ietf.org/rfc/rfc2131.txt
+if(request_packet_type == DHCP_REQUEST_RENEWING) {
+build_udp_ip_headers(dhcp_len, yip_n, sip_n);
+}
+else {
+build_udp_ip_headers(dhcp_len, INADDR_ANY, INADDR_BROADCAST);
+}
 }

 void dhcp_mbuf::compose_release(struct ifnet* ifp,
@@ -420,6 +447,9 @@ namespace dhcp {
 options += op_len;
 }

+dhcp_i( "Received %s message from DHCP server: %s regarding  
offerred IP address: %s",
+dhcp_message_type_name_by_id[_message_type],  
_dhcp_server_ip.to_string().c_str(),

+_your_ip.to_string().c_str());
 return true;
 }

@@ -519,6 +549,7 @@ namespace dhcp {
 // Save transaction id & send
 _xid = dm.get_xid();
 _client_addr = _server_addr = ipv4_zero;
+dhcp_i( "Broadcasting DHCPDISCOVER message with xid: [%d]",_xid);
 _sock->dhcp_send(dm);
 }

@@ -533,6 +564,8 @@ namespace dhcp {

 // Save transaction id & send
 _xid = dm.get_xid();
+dhcp_i( "Unicasting DHCPRELEASE message with xid: [%d] from  
client: %s to server: %s",
+_xid, _client_addr.to_string().c_str(),  
_server_addr.to_string().c_str());

 _sock->dhcp_send(dm);
 // IP and routes have to be removed
 osv::stop_if(_ifp->if_xname, _client_addr.to_string().c_str());
@@ -553,9 +586,13 @@ namespace dhcp {
 _xid = rand();
 dm.compose_request(_ifp,
_xid,
-   _client_addr, _server_addr);
+   _client_addr,
+   _server_addr,
+   dhcp_mbuf::DHCP_REQUEST_RENEWING);

 // Send
+

[COMMIT osv master] math: disable optimizations on files which gcc 4.8.4 miscompiles

[Commit Bot]

From: Nadav Har'El 
Committer: Nadav Har'El 
Branch: master

math: disable optimizations on files which gcc 4.8.4 miscompiles

It appears (see details in issue #867) that one of the optimizations in
Gcc 4.8.4's "-O1" miscompiles code which uses a function called "round()"
to call "lround()" instead. This causes our lround() function (which
calls our round()) to go into a silly infinite loop.

Sadly, it seems that none of the specific "-fno-*" options to disable
specific optimizations avoid this optimization. So the workaround in this
patch is to use -O0 on the affected source files - those which implement
the various {l,ll}round{,f,l} functions. This will slow these functions
by a little bit, but hopefully they are not heavily used by performance-
minded applications, anyway.

Fixes #867.

Signed-off-by: Nadav Har'El 
Message-Id: <20170328203039.18789-1-...@scylladb.com>

---
diff --git a/Makefile b/Makefile
--- a/Makefile
+++ b/Makefile
@@ -1284,6 +1284,20 @@ musl += math/trunc.o
 musl += math/truncf.o
 musl += math/truncl.o

+# Issue #867: Gcc 4.8.4 has a bug where it optimizes the trivial round-
+# related functions incorrectly - it appears to convert calls to any
+# function called round() to calls to a function called lround() -
+# and similarly for roundf() and roundl().
+# None of the specific "-fno-*" options disable this buggy optimization,
+# unfortunately. The simplest workaround is to just disable optimization
+# for the affected files.
+$(out)/musl/src/math/lround.o: conf-opt := $(conf-opt) -O0
+$(out)/musl/src/math/lroundf.o: conf-opt := $(conf-opt) -O0
+$(out)/musl/src/math/lroundl.o: conf-opt := $(conf-opt) -O0
+$(out)/musl/src/math/llround.o: conf-opt := $(conf-opt) -O0
+$(out)/musl/src/math/llroundf.o: conf-opt := $(conf-opt) -O0
+$(out)/musl/src/math/llroundl.o: conf-opt := $(conf-opt) -O0
+
 musl += misc/a64l.o
 libc += misc/basename.o
 musl += misc/dirname.o

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] gdb: fix error on connect on older distros

[Commit Bot]

From: Nadav Har'El 
Committer: Nadav Har'El 
Branch: master

gdb: fix error on connect on older distros

In commit adf78de4, I noticed that to access values of an enum class,
we needed to add the class's name, which we previously didn't.
I speculated that is only needed with newer gdb. I just confirmed that
indeed, if one tries to debug OSv on an old distro (but still new enough
to compile OSv), the new names do not work and we need to use the old ones
from before commit adf78de4 - otherwise we get errors on "connect".

So in this patch we try both names, trying the new names first, and if
they fail, the old names.

Signed-off-by: Nadav Har'El 
Message-Id: <20170315184400.25885-1-...@scylladb.com>

---
diff --git a/scripts/loader.py b/scripts/loader.py
--- a/scripts/loader.py
+++ b/scripts/loader.py
@@ -180,10 +180,16 @@ def invoke(self, arg, from_tty):
 host_port = arg.split()[0]
 gdb.execute('target remote %s' % host_port)
 global status_enum
-status_enum.running =  
gdb.parse_and_eval('sched::thread::status::running')
-status_enum.waiting =  
gdb.parse_and_eval('sched::thread::status::waiting')
-status_enum.queued =  
gdb.parse_and_eval('sched::thread::status::queued')
-status_enum.waking =  
gdb.parse_and_eval('sched::thread::status::waking')

+try:
+status_enum.running =  
gdb.parse_and_eval('sched::thread::status::running')
+status_enum.waiting =  
gdb.parse_and_eval('sched::thread::status::waiting')
+status_enum.queued =  
gdb.parse_and_eval('sched::thread::status::queued')
+status_enum.waking =  
gdb.parse_and_eval('sched::thread::status::waking')

+except:
+status_enum.running =  
gdb.parse_and_eval('sched::thread::running')
+status_enum.waiting =  
gdb.parse_and_eval('sched::thread::waiting')
+status_enum.queued =  
gdb.parse_and_eval('sched::thread::queued')
+status_enum.waking =  
gdb.parse_and_eval('sched::thread::waking')



 Connect()

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] gdb: fix "osv mmap" on connect on older distros

[Commit Bot]

From: Nadav Har'El 
Committer: Nadav Har'El 
Branch: master

gdb: fix "osv mmap" on connect on older distros

In commit ce177a50, I fixed "osv mmap" on newer distros as gdb started
requiring quotes around the typename mmu::vma. Unfortunately, these quotes
are considered a syntax error on older gdb. So in this patch we try both
syntaxes, trying the new one first, and if they it fails, the old one.

Signed-off-by: Nadav Har'El 
Message-Id: <20170316083910.27964-1-...@scylladb.com>

---
diff --git a/scripts/loader.py b/scripts/loader.py
--- a/scripts/loader.py
+++ b/scripts/loader.py
@@ -287,7 +287,10 @@ def vma_list(node=None):
 node = intrusive_set_root_node(fpr)

 if node:
-offset =  
gdb.parse_and_eval("(int)&(('mmu::vma'*)0)->_vma_list_hook")

+try:
+offset =  
gdb.parse_and_eval("(int)&(('mmu::vma'*)0)->_vma_list_hook")

+except:
+offset =  
gdb.parse_and_eval("(int)&((mmu::vma*)0)->_vma_list_hook")

 vma = node.cast(gdb.lookup_type('void').pointer()) - offset
 vma = vma.cast(gdb.lookup_type('mmu::vma').pointer())

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] SIOCGIFFLAGS ioctl: remove BSD-specific bits

[Commit Bot]

From: Nadav Har'El 
Committer: Nadav Har'El 
Branch: master

SIOCGIFFLAGS ioctl: remove BSD-specific bits

The SIOCGIFFLAGS ioctl returns a set of flags for a network interface.
Many of the classic bits there, like IFF_UP and IFF_LOOPBACK, are the same
in our internal implementation (BSD-based) and desired output (Linux),
but not all of them are the same. We already had a function to convert
the internal bitmask into a Linux bitmask, but it was incomplete.

Of particular importance is BSD's IFF_SIMPLEX flag, which does not exist
on Linux, and is set by OSv's Xen network driver. As we failed to clear
this flag, clients thought they were seeing Linux's IFF_SLAVE (which is
at the same bit position, 0x800). In particular OpenMPI did not like
seeing this flag, and refused to use this network interface.

Fixes #862

Signed-off-by: Nadav Har'El 
Message-Id: <20170314133315.18971-1-...@scylladb.com>

---
diff --git a/bsd/sys/compat/linux/linux_ioctl.cc  
b/bsd/sys/compat/linux/linux_ioctl.cc

--- a/bsd/sys/compat/linux/linux_ioctl.cc
+++ b/bsd/sys/compat/linux/linux_ioctl.cc
@@ -143,26 +143,27 @@ linux_ifconf(struct bsd_ifconf *ifc_p)
 return (0);
 }

+// While many of the traditional bits returned by SIOCGIFFLAGS are the same
+// on BSD and Linux, some of the newer ones have different location, or
+// different meaning for the same bit location. So we need to convert the
+// BSD bits which we store internally to the Linux bits applications expect
+// us to return.
 static void
 linux_gifflags(struct ifnet *ifp, struct l_ifreq *ifr)
 {
 l_short flags;

+// This assignment drops all the flags beyond the 16th bit.
+// None of them have a Linux equivalent.
 flags = (ifp->if_flags | ifp->if_drv_flags);
-/* These flags have no Linux equivalent
- *
- *  Notes:
- *   - We do show IFF_SMART|IFF_DRV_OACTIVE|IFF_SIMPLEX
- *   - IFF_LINK0 has a value of 0x1000 which conflics with the  
Linux

- * IFF_MULTICAST value.
- */
-flags &= ~(IFF_LINK0|IFF_LINK1|IFF_LINK2);
+// These flags have no Linux equivalent:
+flags &= ~(IFF_SMART|IFF_DRV_OACTIVE|IFF_SIMPLEX|IFF_LINK0|IFF_LINK1| 
IFF_LINK2);

 /* Linux' multicast flag is in a different bit */
 if (flags & IFF_MULTICAST) {
 flags &= ~IFF_MULTICAST;
 flags |= 0x1000;
 }
-ifr->ifr_flags = flags ;
+ifr->ifr_flags = flags;
 }

 #define ARPHRD_ETHER   1

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] osv: xen: independent event interrupt bring up

[Commit Bot]

From: Sergiy Kibrik 
Committer: Nadav Har'El 
Branch: master

osv: xen: independent event interrupt bring up

Initialize Xen event interrupt separately from xenbus driver, so that events
can be served before xenbus is up, or even in case when xenbus not being  
probed

at all or not required, e.g. in case of aarch64 Xen console.

Signed-off-by: Sergiy Kibrik 
Message-Id:  
<1490017967-16388-1-git-send-email-sergiy.kib...@globallogic.com>


---
diff --git a/arch/aarch64/xen.cc b/arch/aarch64/xen.cc
--- a/arch/aarch64/xen.cc
+++ b/arch/aarch64/xen.cc
@@ -8,6 +8,9 @@
 #include 
 #include 
 #include 
+#include  /* __dead2 defined here */
+#include 
+#include 

 #include "arch-dtb.hh"

@@ -17,6 +20,20 @@ namespace xen {

 shared_info_t dummy_info;
 struct xen_shared_info xen_shared_info __attribute__((aligned(4096)));
+constexpr int events_irq = 31; /*FIXME: get from FDT */
+
+void irq_init()
+{
+if (!is_xen())
+return;
+
+evtchn_init(NULL);
+
+auto intr = new spi_interrupt(gic::irq_type::IRQ_TYPE_LEVEL,  
events_irq,

+  xen::xen_ack_irq,
+  xen::xen_handle_irq);
+irq_setup(intr);
+}

 }

diff --git a/arch/x64/xen.cc b/arch/x64/xen.cc
--- a/arch/x64/xen.cc
+++ b/arch/x64/xen.cc
@@ -16,6 +16,7 @@
 #include 
 #include 
 #include 
+#include 

 shared_info_t *HYPERVISOR_shared_info;
 uint8_t xen_features[XENFEAT_NR_SUBMAPS * 32];
@@ -201,6 +202,20 @@ void xen_init(processor::features_type ,  
unsigned base)
 HYPERVISOR_shared_info = reinterpret_cast*>(_shared_info);

 }

+void irq_init()
+{
+if (!is_xen())
+return;
+
+evtchn_init(NULL);
+
+/* if vector callback not supported, platform PCI driver will handle  
that */

+if (processor::features().xen_vector_callback)
+xen::xen_set_callback();
+
+irq_setup(NULL);
+}
+
 extern "C"
 void xen_init(struct start_info* si)
 {
diff --git a/bsd/sys/xen/evtchn.h b/bsd/sys/xen/evtchn.h
--- a/bsd/sys/xen/evtchn.h
+++ b/bsd/sys/xen/evtchn.h
@@ -32,6 +32,8 @@ void unmask_evtchn(int port);

 int evtchn_from_irq(int irq);

+void evtchn_init(void *arg);
+
 #ifdef SMP
 void rebind_evtchn_to_cpu(int port, unsigned int cpu);
 #else
diff --git a/core/xen_intr.cc b/core/xen_intr.cc
--- a/core/xen_intr.cc
+++ b/core/xen_intr.cc
@@ -111,9 +111,11 @@ void xen_irq::_cpu_init(sched::cpu *c)
 (*(_thread.for_cpu(c)))->start();
 }

-xen_irq::xen_irq()
+xen_irq::xen_irq(interrupt *intr)
 : _cpu_notifier([this] { cpu_init(); })
 {
+if (intr)
+_intr.reset(intr);
 }

 static xen_irq *xen_irq_handlers;
@@ -129,13 +131,10 @@ bool xen_ack_irq()
 return true;
 }

-static __attribute__((constructor)) void setup_xen_irq()
+void irq_setup(interrupt *intr)
 {
-if (!is_xen()) {
-return;
-}
-
-xen_irq_handlers = new xen_irq;
+assert(is_xen());
+xen_irq_handlers = new xen_irq(intr);
 }
 }

diff --git a/drivers/xenfront-xenbus.cc b/drivers/xenfront-xenbus.cc
--- a/drivers/xenfront-xenbus.cc
+++ b/drivers/xenfront-xenbus.cc
@@ -26,7 +26,6 @@
 int xs_attach(struct device *);

 int xenpci_irq_init(device_t device, struct xenpci_softc *scp);
-void evtchn_init(void *arg);

 int xenbusb_front_probe(device_t dev);
 int xenbusb_front_attach(device_t dev);
@@ -64,13 +63,7 @@ xenbus::xenbus(pci::device& pci_dev)
 _dev.set_bus_master(true);
 _driver_name = std::string("xenfront-xenbus");

-// From here on, all the OSV details are sorted, and we start the Xen
-// bringup
-evtchn_init(NULL);
-
-if (processor::features().xen_vector_callback) {
-xen::xen_set_callback();
-} else {
+if (!processor::features().xen_vector_callback) {
 _pgsi.reset(xen::xen_set_callback(irqno));
 }

diff --git a/include/osv/xen.hh b/include/osv/xen.hh
--- a/include/osv/xen.hh
+++ b/include/osv/xen.hh
@@ -16,6 +16,7 @@
 #include 
 #include 
 #include 
+#include 

 extern char hypercall_page[];
 extern uint8_t xen_features[];
@@ -53,7 +54,8 @@ extern struct xen_shared_info xen_shared_info;
 void xen_set_callback();
 void xen_handle_irq();
 bool xen_ack_irq();
-
+void irq_setup(interrupt *intr);
+void irq_init();
 }


diff --git a/include/osv/xen_intr.hh b/include/osv/xen_intr.hh
--- a/include/osv/xen_intr.hh
+++ b/include/osv/xen_intr.hh
@@ -14,7 +14,7 @@ namespace xen {

 class xen_irq {
 public:
-explicit xen_irq();
+explicit xen_irq(interrupt *intr);
 void wake() { (*_thread)->wake(); }
 static void register_irq(int vector, driver_intr_t handler, void *arg);
 static void unregister_irq(int vector);
@@ -24,6 +24,8 @@ private:
 sched::cpu::notifier _cpu_notifier;
 void _cpu_init(sched::cpu *c);

+std::unique_ptr _intr;
+
 static percpu  _thread;
 };
 }
diff --git a/loader.cc b/loader.cc
--- a/loader.cc
+++ b/loader.cc
@@ -44,6 +44,7 @@
 #include 
 #include 
 #include 
+#include 
 #include 
 #include 
 

[COMMIT osv master] osv: xen: split xenbus into generic & PCI parts

[Commit Bot]

From: Sergiy Kibrik 
Committer: Nadav Har'El 
Branch: master

osv: xen: split xenbus into generic & PCI parts

Xen platform PCI is x86-specific feature, not currently present on aarch64.

For xenbus to work on both platforms the generic xenbus driver should be  
probed
directly (under aarch64 in case Xen is detected), but in case of x64  
special PCI

driver checks for Xen platform PCI device and probes generic xenbus driver
if such device is found, pretty much like the way it is done in Linux.

Signed-off-by: Sergiy Kibrik 
Message-Id:  
<1490018087-16439-1-git-send-email-sergiy.kib...@globallogic.com>


---
diff --git a/Makefile b/Makefile
--- a/Makefile
+++ b/Makefile
@@ -818,6 +818,7 @@ drivers += drivers/ahci.o
 drivers += drivers/ide.o
 drivers += drivers/scsi-common.o
 drivers += drivers/vmw-pvscsi.o
+drivers += drivers/xenplatform-pci.o
 endif # x64

 ifeq ($(arch),aarch64)
diff --git a/arch/x64/arch-setup.cc b/arch/x64/arch-setup.cc
--- a/arch/x64/arch-setup.cc
+++ b/arch/x64/arch-setup.cc
@@ -255,7 +255,7 @@ void arch_init_premain()
 #include "drivers/virtio-net.hh"
 #include "drivers/virtio-assign.hh"
 #include "drivers/virtio-rng.hh"
-#include "drivers/xenfront-xenbus.hh"
+#include "drivers/xenplatform-pci.hh"
 #include "drivers/ahci.hh"
 #include "drivers/vmw-pvscsi.hh"
 #include "drivers/vmxnet3.hh"
@@ -283,7 +283,7 @@ void arch_init_drivers()
 drvman->register_driver(virtio::net::probe);
 }
 drvman->register_driver(virtio::rng::probe);
-drvman->register_driver(xenfront::xenbus::probe);
+drvman->register_driver(xenfront::xenplatform_pci::probe);
 drvman->register_driver(ahci::hba::probe);
 drvman->register_driver(vmw::pvscsi::probe);
 drvman->register_driver(vmw::vmxnet3::probe);
diff --git a/drivers/xenfront-xenbus.cc b/drivers/xenfront-xenbus.cc
--- a/drivers/xenfront-xenbus.cc
+++ b/drivers/xenfront-xenbus.cc
@@ -5,8 +5,6 @@
  * BSD license as described in the LICENSE file in the top-level directory.
  */

-#include "msr.hh"
-#include "xen.hh"
 #include 
 #include 
 #include "string.h"
@@ -46,27 +44,17 @@ namespace xenfront {

 xenbus *xenbus::_instance = nullptr;

-xenbus::xenbus(pci::device& pci_dev)
+xenbus::xenbus()
 : hw_driver()
-, _dev(pci_dev)
 {
-int irqno = pci_dev.get_interrupt_line();
-
 if (_instance) {
 return;
 } else {
 _instance = this;
 }

-parse_pci_config();
-
-_dev.set_bus_master(true);
 _driver_name = std::string("xenfront-xenbus");

-if (!processor::features().xen_vector_callback) {
-_pgsi.reset(xen::xen_set_callback(irqno));
-}
-
 xs_attach(&_xenstore_device);

 xs_scanf(XST_NIL, "domid", "", NULL, "%d", &_domid);
@@ -127,27 +115,14 @@ void  
xenbus::for_each_child(std::function func)


 hw_driver* xenbus::probe(hw_device* dev)
 {
-if (!processor::features().xen_pci) {
+if (!is_xen())
 return nullptr;
-}
-
-if (auto pci_dev = dynamic_cast(dev)) {
-// dev id is the same for all xen devices?
-if (pci_dev->get_subsystem_vid() == XEN_VENDOR_ID) {
-return new xenbus(*pci_dev);
-}
-}
-return nullptr;
+return new xenbus();
 }

 void xenbus::dump_config()
 {
-_dev.dump_config();
-}
-
-bool xenbus::parse_pci_config()
-{
-return true;
+/*TODO: print type, name and node path */
 }
 };

diff --git a/drivers/xenfront-xenbus.hh b/drivers/xenfront-xenbus.hh
--- a/drivers/xenfront-xenbus.hh
+++ b/drivers/xenfront-xenbus.hh
@@ -9,22 +9,18 @@
 #define XENFRONT_BUS_DRIVER_H

 #include "drivers/xenfront.hh"
-#include "drivers/pci-device.hh"
+#include "drivers/device.hh"
 #include 
 #include 

 namespace xenfront {
-
 class xenbus : public hw_driver {
 public:

-explicit xenbus(pci::device& dev);
+explicit xenbus();
 static hw_driver* probe(hw_device* dev);
-pci::device& pci_device() { return _dev; }
-
-bool parse_pci_config();
-void dump_config();

+virtual void dump_config();
 virtual std::string get_name() const { return _driver_name; }
 const std::string _node_path() { return _node_path; }

@@ -40,8 +36,6 @@ namespace xenfront {
 private:
 static struct xenbus *_instance;
 void wait_for_devices();
-pci::device& _dev;
-std::unique_ptr _pgsi;
 struct device _xenstore_device;

 std::vector _children;
diff --git a/drivers/xenfront.hh b/drivers/xenfront.hh
--- a/drivers/xenfront.hh
+++ b/drivers/xenfront.hh
@@ -18,8 +18,6 @@
 #include 
 #include 

-#define XEN_VENDOR_ID 0x5853
-
 struct xenbus_device_ivars;

 template 
diff --git a/drivers/xenplatform-pci.cc b/drivers/xenplatform-pci.cc
--- a/drivers/xenplatform-pci.cc
+++ b/drivers/xenplatform-pci.cc
@@ -0,0 +1,41 @@
+/*
+ * Copyright (C) 2013 Cloudius Systems, Ltd.
+ *   2017 

[COMMIT osv master] aarch64: mmu: dispose of device address ranges

[Commit Bot]

From: Sergiy Kibrik' via OSv Development 
Committer: Nadav Har'El 
Branch: master

aarch64: mmu: dispose of device address ranges

Each ARM SOCs (System On Chip) family have different address space division
into device and regular memory. So hard-coded ranges forbids passing device  
IO

range directly to guest VM.

Also shared Xen pages (console and xenstore shared rings) reside in this
hard-coded device range, so can't be mapped without mangling ranges.

Provided all the above, there seem to be not much value in keeping this
check -- particularly after mattr introduction and OS extention for Xen  
support.


Signed-off-by: Sergiy Kibrik 
Message-Id: <1488534351-8263-7-git-send-email-sergiy.kib...@globallogic.com>

---
diff --git a/arch/aarch64/arch-mmu.hh b/arch/aarch64/arch-mmu.hh
--- a/arch/aarch64/arch-mmu.hh
+++ b/arch/aarch64/arch-mmu.hh
@@ -18,9 +18,6 @@

 namespace mmu {
 constexpr int max_phys_addr_size = 48;
-// device_range_* are used only for debug purpose
-constexpr int device_range_start = 0x300;
-constexpr int device_range_stop = 0x4000;
 extern u64 mem_addr; /* set by the dtb_setup constructor */

 enum class mattr {
@@ -144,11 +141,6 @@ inline void pt_element_common::set_pfn(u64 pfn,  
bool large) {

 set_addr(pfn << page_size_shift, large);
 }

-static inline bool dbg_mem_is_dev(phys addr)
-{
-return addr >= mmu::device_range_start && addr <  
mmu::device_range_stop;

-}
-
 template
 pt_element make_pte(phys addr, bool leaf, unsigned perm = perm_rwx,
mattr mem_attr = mattr_default)
@@ -172,11 +164,9 @@ pt_element make_pte(phys addr, bool leaf, unsigned  
perm = perm_rwx,

 switch (mem_attr) {
 default:
 case mattr::normal:
-assert(!dbg_mem_is_dev(addr));
 pte.set_attridx(4);
 break;
 case mattr::dev:
-assert(dbg_mem_is_dev(addr));
 pte.set_attridx(0);
 break;
 }

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] bsd: xen: extend features.h with additional definitions

[Commit Bot]

From: Sergiy Kibrik' via OSv Development 
Committer: Nadav Har'El 
Branch: master

bsd: xen: extend features.h with additional definitions

Signed-off-by: Sergiy Kibrik 
Message-Id: <1488534351-8263-4-git-send-email-sergiy.kib...@globallogic.com>

---
diff --git a/bsd/sys/xen/interface/features.h  
b/bsd/sys/xen/interface/features.h

--- a/bsd/sys/xen/interface/features.h
+++ b/bsd/sys/xen/interface/features.h
@@ -59,6 +59,33 @@
 /* x86: Does this Xen host support the MMU_PT_UPDATE_PRESERVE_AD  
hypercall? */

 #define XENFEAT_mmu_pt_update_preserve_ad  5

+/* x86: Does this Xen host support the MMU_{CLEAR,COPY}_PAGE hypercall? */
+#define XENFEAT_highmem_assist 6
+
+/*
+ * If set, GNTTABOP_map_grant_ref honors flags to be placed into guest  
kernel

+ * available pte bits.
+ */
+#define XENFEAT_gnttab_map_avail_bits  7
+
+/* x86: Does this Xen host support the HVM callback vector type? */
+#define XENFEAT_hvm_callback_vector8
+
+/* x86: pvclock algorithm is safe to use on HVM */
+#define XENFEAT_hvm_safe_pvclock   9
+
+/* x86: pirq can be used by HVM guests */
+#define XENFEAT_hvm_pirqs 10
+
+/* operation as Dom0 is supported */
+#define XENFEAT_dom0  11
+
+/* Xen also maps grant references at pfn = mfn.
+ * This feature flag is deprecated and should not be used.
+#define XENFEAT_grant_map_identity12
+ */
+
+
 #define XENFEAT_NR_SUBMAPS 1

 #endif /* __XEN_PUBLIC_FEATURES_H__ */

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] osv: aarch64: support processor::features()

[Commit Bot]

From: Sergiy Kibrik 
Committer: Nadav Har'El 
Branch: master

osv: aarch64: support processor::features()

Features used by various Xen drivers, i.e. xenbus.

Signed-off-by: Sergiy Kibrik 
Reviewed-by: Nadav Har'El 
Message-Id:  
<1490090836-18154-1-git-send-email-sergiy.kib...@globallogic.com>


---
diff --git a/arch/aarch64/cpuid.cc b/arch/aarch64/cpuid.cc
--- a/arch/aarch64/cpuid.cc
+++ b/arch/aarch64/cpuid.cc
@@ -7,6 +7,7 @@

 #include "cpuid.hh"
 #include "processor.hh"
+#include "xen.hh"

 namespace processor {

@@ -76,4 +77,21 @@ const std::string& features_str()
 return cpuid_str;
 }

+void process_cpuid(features_type& features)
+{
+xen::get_features(features);
+}
+
+const features_type& features()
+{
+// features() can be used very early, make sure it is initialized
+static features_type f;
+return f;
+}
+
+features_type::features_type()
+{
+process_cpuid(*this);
+}
+
 }
diff --git a/arch/aarch64/cpuid.hh b/arch/aarch64/cpuid.hh
--- a/arch/aarch64/cpuid.hh
+++ b/arch/aarch64/cpuid.hh
@@ -28,6 +28,15 @@ enum hwcap_bit {
 HWCAP_BIT_N
 };

+struct features_type {
+features_type();
+bool xen_clocksource;
+bool xen_vector_callback;
+bool xen_pci;
+};
+
+extern const features_type& features();
+
 }

 #endif /* CPUID_HH_ */
diff --git a/arch/aarch64/xen.cc b/arch/aarch64/xen.cc
--- a/arch/aarch64/xen.cc
+++ b/arch/aarch64/xen.cc
@@ -8,20 +8,48 @@
 #include 
 #include 
 #include 
+#include 
 #include  /* __dead2 defined here */
 #include 
 #include 

 #include "arch-dtb.hh"
+#include "cpuid.hh"

 shared_info_t *HYPERVISOR_shared_info;
+uint8_t xen_features[XENFEAT_NR_SUBMAPS * 32];

 namespace xen {

 shared_info_t dummy_info;
 struct xen_shared_info xen_shared_info __attribute__((aligned(4096)));
 constexpr int events_irq = 31; /*FIXME: get from FDT */

+/*TODO: this can be common x64/aarch64 code */
+void get_features(processor::features_type )
+{
+if (!is_xen())
+return;
+
+for (unsigned int i = 0; i < XENFEAT_NR_SUBMAPS; i++) {
+struct xen_feature_info info = {
+.submap_idx = i,
+};
+
+if (HYPERVISOR_xen_version(XENVER_get_features, ) < 0)
+assert(0);
+
+for (int j = 0; j < 32; j++)
+xen_features[i * 32 + j] = !!(info.submap & 1<

[COMMIT osv master] osv: aarch64: map Xen shared_info page

[Commit Bot]

From: Sergiy Kibrik 
Committer: Nadav Har'El 
Branch: master

osv: aarch64: map Xen shared_info page

Shared structure is used by various protocols communicating with hypervisor
(e.g. for events processing).

Signed-off-by: Sergiy Kibrik 
Message-Id:  
<1490090836-18154-2-git-send-email-sergiy.kib...@globallogic.com>


---
diff --git a/arch/aarch64/xen.cc b/arch/aarch64/xen.cc
--- a/arch/aarch64/xen.cc
+++ b/arch/aarch64/xen.cc
@@ -12,6 +12,7 @@
 #include  /* __dead2 defined here */
 #include 
 #include 
+#include 

 #include "arch-dtb.hh"
 #include "cpuid.hh"
@@ -21,8 +22,7 @@ uint8_t xen_features[XENFEAT_NR_SUBMAPS * 32];

 namespace xen {

-shared_info_t dummy_info;
-struct xen_shared_info xen_shared_info __attribute__((aligned(4096)));
+struct xen_shared_info xen_shared_info __attribute__((aligned(PAGE_SIZE)));
 constexpr int events_irq = 31; /*FIXME: get from FDT */

 /*TODO: this can be common x64/aarch64 code */
@@ -50,6 +50,21 @@ void get_features(processor::features_type )
 evtchn_irq_is_legacy();
 }

+void setup()
+{
+   struct xen_add_to_physmap map = {
+.domid = DOMID_SELF,
+.space = XENMAPSPACE_shared_info,
+.idx = 0,
+.gpfn = reinterpret_cast(_shared_info) >>  
PAGE_SHIFT,

+};
+
+if (HYPERVISOR_memory_op(XENMEM_add_to_physmap, ))
+assert(0);
+
+HYPERVISOR_shared_info = reinterpret_cast*>(_shared_info);

+}
+
 void irq_init()
 {
 if (!is_xen())
@@ -70,11 +85,8 @@ extern "C" {
 void init_xen()
 {
 HYPERVISOR_shared_info = nullptr;
-if (dtb_get_vmm_is_xen()) {
-/* set valid pointer just to know we're under Xen.
- * Real shared page will be set up later, when page allocator  
works.

-*/
-HYPERVISOR_shared_info = ::dummy_info;
-}
+if (dtb_get_vmm_is_xen())
+xen::setup();
 }
+
 }

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] build: aarch64: eliminate warnings from imgedit.py

[Commit Bot]

From: Nadav Har'El 
Committer: Nadav Har'El 
Branch: master

build: aarch64: eliminate warnings from imgedit.py

Commit 6703d2283158f062eb0bccf0db9ccb73c6bbe6ac eliminated some annoying
complaints by newer qemu that we didn't explicitly specify the format of
a raw image, but I forgot to fix one place specific to the aarch64 build -
so this patch fixes that place.

Signed-off-by: Nadav Har'El 

---
diff --git a/Makefile b/Makefile
--- a/Makefile
+++ b/Makefile
@@ -487,7 +487,7 @@ $(out)/preboot.bin: $(out)/preboot.elf
 $(out)/loader.img: $(out)/preboot.bin $(out)/loader-stripped.elf
 	$(call quiet, dd if=$(out)/preboot.bin of=$@ > /dev/null 2>&1, DD $@  
preboot.bin)
 	$(call quiet, dd if=$(out)/loader-stripped.elf of=$@ conv=notrunc  
obs=4096 seek=16 > /dev/null 2>&1, DD $@ loader-stripped.elf)

-   $(call quiet, scripts/imgedit.py setargs $@ $(cmdline), IMGEDIT $@)
+	$(call quiet, scripts/imgedit.py setargs "-f raw $@" $(cmdline), IMGEDIT  
$@)


 endif # aarch64

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] xenconsole: shared ring suport

[Commit Bot]

From: Sergiy Kibrik 
Committer: Nadav Har'El 
Branch: master

xenconsole: shared ring suport

Complete driver to properly work with xl console utility, i.e. full  
read/write
access. Driver is extended to work with shared structure (ring) and event  
channel.


Signed-off-by: Sergiy Kibrik 
Message-Id:  
<1490018156-16491-1-git-send-email-sergiy.kib...@globallogic.com>


---
diff --git a/drivers/xenconsole.cc b/drivers/xenconsole.cc
--- a/drivers/xenconsole.cc
+++ b/drivers/xenconsole.cc
@@ -5,32 +5,105 @@
  * BSD license as described in the LICENSE file in the top-level directory.
  */

+#define XENHVM //FIXME hack to reveal hvm_get_parameter()
 #include  /* __dead2 defined here */
+#include 
+#include 
 #include 
-
+#include 
+#include 
+#include 
+#include 
+#include 
 #include "xenconsole.hh"

 namespace console {

+XEN_Console::
+XEN_Console::XEN_Console()
+: _interface(0)
+ , _evtchn(-1)
+ , _irq(0)
+ , _pfn(0)
+{}
+
+void XEN_Console::handle_intr()
+{
+_thread->wake();
+}
+
 void XEN_Console::write(const char *str, size_t len) {
-   HYPERVISOR_console_write(str, len);
+assert(len > 0);
+if (!_interface) {
+HYPERVISOR_console_write(str, len);
+return;
+}
+
+/* str might be larger then ring, so write it by chunks in this case */
+XENCONS_RING_IDX prod = _interface->out_prod;
+constexpr auto ringsize = sizeof(_interface->out) - 1;
+while (true) {
+XENCONS_RING_IDX cons = _interface->out_cons;
+auto delta = prod - cons;
+
+if (unlikely(delta > ringsize)) {
+prod = cons; /* ring is corrupted, reset is the best we can do  
*/

+delta = 0;
+}
+
+size_t c = 0;
+for (; c < std::min(ringsize - delta, len); c++)
+_interface->out[MASK_XENCONS_IDX(prod++, _interface->out)] =  
*str++;

+
+_interface->out_prod = prod;
+wmb();
+
+if (likely(c == len))
+break; /* flush() will do evtchn notification */
+
+len -= ringsize - delta;
+notify_remote_via_evtchn(_evtchn);
+
+while (_interface->out_cons != _interface->out_prod)
+cpu_relax(); /* can't sleep here */
+}
 }

 void XEN_Console::dev_start()
 {
+_pfn = hvm_get_parameter(HVM_PARAM_CONSOLE_PFN);
+_evtchn = hvm_get_parameter(HVM_PARAM_CONSOLE_EVTCHN);
+
+if (!_pfn || !_evtchn)
+throw std::runtime_error("fail to get console params");
+
+ if (bind_caller_port_to_irqhandler(_evtchn, "xenconsole",
+XEN_Console::console_intr,
+static_cast(this),
+INTR_TYPE_MISC, &_irq) != 0)
+throw std::runtime_error("fail to bind evtchn");
+
+_interface = (xencons_interface*)mmio_map(_pfn << PAGE_SHIFT,  
PAGE_SIZE);

 }

 void XEN_Console::flush()
 {
+notify_remote_via_evtchn(_evtchn);
 }

 bool XEN_Console::input_ready()
 {
-   return false; /*TODO: support input */
+return _interface->in_cons != _interface->in_prod;
 }

 char XEN_Console::readch() {
-return '\0'; /*TODO: support input */
+XENCONS_RING_IDX cons;
+char c;
+assert(_interface);
+cons = _interface->in_cons;
+c = _interface->in[MASK_XENCONS_IDX(cons, _interface->in)];
+mb();
+_interface->in_cons = cons + 1;
+return c;
 }
-
 }
diff --git a/drivers/xenconsole.hh b/drivers/xenconsole.hh
--- a/drivers/xenconsole.hh
+++ b/drivers/xenconsole.hh
@@ -9,13 +9,14 @@
 #define XEN_CONSOLE_HH

 #include "console-driver.hh"
-#include "exceptions.hh"
-#include 
+#include 
+#include 

 namespace console {

 class XEN_Console : public console_driver {
 public:
+XEN_Console();
 virtual void write(const char *str, size_t len);
 virtual void flush();
 virtual bool input_ready();
@@ -24,6 +25,15 @@ public:
 private:
 virtual void dev_start();
 virtual const char *thread_name() { return "xen-input"; }
+void handle_intr();
+static void console_intr(void *arg) {
+static_cast(arg)->handle_intr();
+}
+
+struct xencons_interface *_interface;
+int _evtchn;
+unsigned int _irq;
+unsigned long _pfn;
 };

 }

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] bsd: xen: add console parameters

[Commit Bot]

From: Sergiy Kibrik' via OSv Development 
Committer: Nadav Har'El 
Branch: master

bsd: xen: add console parameters

Add standard console HVM interface parameters. They're intended for use by
console driver to get shared ring page and event channel No. from Xen.

Signed-off-by: Sergiy Kibrik 
Message-Id: <1488534351-8263-8-git-send-email-sergiy.kib...@globallogic.com>

---
diff --git a/bsd/sys/xen/interface/hvm/params.h  
b/bsd/sys/xen/interface/hvm/params.h

--- a/bsd/sys/xen/interface/hvm/params.h
+++ b/bsd/sys/xen/interface/hvm/params.h
@@ -56,6 +56,9 @@
 #define HVM_PARAM_BUFPIOREQ_PFN9
 #endif

+#define HVM_PARAM_CONSOLE_PFN17
+#define HVM_PARAM_CONSOLE_EVTCHN 18
+
 /*
  * Set mode for virtual timers (currently x86 only):
  *  delay_for_missed_ticks (default):

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv-apps master] Update redis-memonly package to version 3.2.8

[Commit Bot]

From: Nadav Har'El 
Committer: Nadav Har'El 
Branch: master

Update redis-memonly package to version 3.2.8

Unfortunately required another small source code patch to redis, which
didn't gracefully handle OSv's total lack of recognition for IPv6.
See issue #865 and upstream issue  
https://github.com/antirez/redis/issues/3894.


Fixes #865.

Signed-off-by: Nadav Har'El 

---
diff --git a/redis-memonly/GET b/redis-memonly/GET
--- a/redis-memonly/GET
+++ b/redis-memonly/GET
@@ -3,7 +3,7 @@
 # Build a directory from the upstream redis
 set -e

-VERSION=3.0.1
+VERSION=3.2.8
 mkdir upstream
 cd upstream
 wget https://github.com/antirez/redis/archive/$VERSION.tar.gz
diff --git a/redis-memonly/patch b/redis-memonly/patch
--- a/redis-memonly/patch
+++ b/redis-memonly/patch
@@ -14,3 +14,14 @@
  /* Byte ordering detection */
  #include  /* This will likely define BYTE_ORDER */

+--- src/server.c   2017-03-22 13:36:14.699146635 +0200
 /tmp/server.c  2017-03-22 13:37:01.809136644 +0200
+@@ -1786,7 +1786,7 @@
+ if (fds[*count] != ANET_ERR) {
+ anetNonBlock(NULL,fds[*count]);
+ (*count)++;
+-} else if (errno == EAFNOSUPPORT) {
++} else if (errno == EAFNOSUPPORT || errno == EPROTONOSUPPORT)  
{

+ unsupported++;
+ serverLog(LL_WARNING,"Not listening to IPv6:  
unsupproted");

+ }

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] Update apps submodule

[Commit Bot]

From: Nadav Har'El 
Committer: Nadav Har'El 
Branch: master

Update apps submodule

Signed-off-by: Nadav Har'El 

---
diff --git a/apps b/apps
--- a/apps
+++ b/apps
@@ -1 +1 @@
-Subproject commit 3bf4a46abb0a95bb9998d1877131ef543ab0abbb
+Subproject commit 82c7913b12e9c11a05ef99a8c2d6ff71df5b2599

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] command line: expand environ vars for runscript and REST commands

[Commit Bot]

From: Justin Cinkelj 
Committer: Nadav Har'El 
Branch: master

command line: expand environ vars for runscript and REST commands

When command is run via CLI (run.py -e "CMD"), the CMD string can contain
$KEY, and KEY will be replaced with corresponding value from environ.
The patch implements this also for commands run via REST or via runscript.

Fixes #868

Signed-off-by: Justin Cinkelj 
Message-Id: <20170329125439.1316-1-justin.cink...@xlab.si>

---
diff --git a/core/commands.cc b/core/commands.cc
--- a/core/commands.cc
+++ b/core/commands.cc
@@ -92,6 +92,32 @@ parse_command_line_min(const std::string line, bool )
 }

 /*
+Everything after first $ is assumed to be env var.
+So with AA=aa, "$AA" -> "aa", and "X$AA" => "Xaa"
+More than one $ is not supported, and "${AA}" is also not.
+*/
+void expand_environ_vars(std::vector& result)
+{
+std::vector::iterator cmd_iter;
+std::vector::iterator line_iter;
+for (cmd_iter=result.begin(); cmd_iter!=result.end(); cmd_iter++) {
+for (line_iter=cmd_iter->begin(); line_iter!=cmd_iter->end();  
line_iter++) {

+size_t pos;
+if ((pos = line_iter->find_first_of('$')) !=  
std::string::npos) {

+std::string new_word = line_iter->substr(0, pos);
+std::string key = line_iter->substr(pos+1);
+auto tmp = getenv(key.c_str());
+//debug("new_word=%s, key=%s, tmp=%s\n",  
new_word.c_str(), key.c_str(), tmp);

+if (tmp) {
+new_word += tmp;
+}
+*line_iter = new_word;
+}
+}
+}
+}
+
+/*
 In each runscript line, first N args starting with - are options.
 Parse options and remove them from result.

@@ -193,6 +219,9 @@ runscript_expand(const std::vector& cmd,  
bool , bool _runscri

 ok = false;
 return result2;
 }
+// Replace env vars found inside script.
+// Options, script command and script command parameters can  
be set via env vars.

+expand_environ_vars(result3);
 // process and remove options from command
 runscript_process_options(result3);
 result2.insert(result2.end(), result3.begin(), result3.end());
@@ -210,6 +239,8 @@ parse_command_line(const std::string line,  bool )
 {
 std::vector result, result2;
 result = parse_command_line_min(line, ok);
+// First replace environ variables in input command line.
+expand_environ_vars(result);

 /*
 If command starts with runscript, we need to read actual command to
diff --git a/loader.cc b/loader.cc
--- a/loader.cc
+++ b/loader.cc
@@ -316,20 +316,7 @@ std::vector  
prepare_commands(int ac, char** av)


 // concatenate everything
 for (auto i = 0; i < ac; i++) {
-std::string arg("");
-char* env = strchr(av[i],'$');
-if (av[i] && env) {
-*env = '\0';
-env++;
-auto tmp = getenv(env);
-arg = av[i];
-if (tmp) {
-arg += tmp;
-}
-} else {
-arg = av[i];
-}
-line += arg + " ";
+line += std::string(av[i]) + " ";
 }

 commands = osv::parse_command_line(line, ok);
diff --git a/tests/tst-commands.cc b/tests/tst-commands.cc
--- a/tests/tst-commands.cc
+++ b/tests/tst-commands.cc
@@ -672,6 +672,91 @@ static bool test_runscript_with_env()
 return false;
 }

+unsetenv("ASDF");
+
+return true;
+}
+
+static bool test_runscript_with_env_in_script()
+{
+std::ofstream of1("/myscript", std::ios::out | std::ios::binary);
+of1 << "--env=ASDF=ttrt --env=PORT=$THEPORT /$THEPROG pp1a $PRM1b pp1c  
$PRM1d\n";

+of1.close();
+
+std::vector result;
+std::vector cmd = { "/prog1" };
+size_t expected_size[] = {6};
+bool ok;
+
+// those two are set during command parsing
+if (NULL != getenv("ASDF")) {
+return false;
+}
+if (NULL != getenv("PORT")) {
+return false;
+}
+// those are required during command parsing
+if (0 != setenv("THEPORT", "4321", 1)) {
+return false;
+}
+if (0 != setenv("THEPROG", "prog1", 1)) {
+return false;
+}
+if (0 != setenv("PRM1b", "pp1b", 1)) {
+return false;
+}
+if (0 != setenv("PRM1d", "pp1d", 1)) {
+return false;
+}
+
+result = osv::parse_command_line(
+std::string("runscript \"/myscript\";  "),
+ok);
+
+if (!ok) {
+return false;
+}
+
+if (result.size() != 1) {
+return false;
+}
+
+for (size_t i = 0; i < result.size(); i++) {
+if (result[i].size() != expected_size[i]) {
+return false;
+}
+if (result[i][0] != cmd[i]) {
+return false;
+}
+ 

[COMMIT osv master] Fix SMP initialization of paravirtual KVM clock

[Commit Bot]

From: Nadav Har'El 
Committer: Nadav Har'El 
Branch: master

Fix SMP initialization of paravirtual KVM clock

Issue #869 reported that the bug which was supposedly fixed by commit
34232de9 has returned when running OSv in a Linux 4.10 host. This patch
fixes the same issue yet again, more correctly, and fixes #869:

For the paravirtual KVM (or Xen) clock, our implementation of uptime()
initially returns 0, until the time that _smp_init is set. Afterwards,
uptime() returns:

 system_time() - _boot_systemtime

The thinking was that when _smp_init is set, so is _boot_systemtime.
That is indeed correct, but the problem is that at this point, system_time()
doesn't necessarily work! The first CPU to call pv_based_clock::setup_cpu()
will set up _boot_systemtime and _smp_init, but at that point other CPUs who
have not yet called setup_cpu() will have a zero system_time(), and as
a result the subtraction above will be negative. So uptime() will return
a negative time on those CPUs, after having previously returned time 0.
Our timer implementation can detect the clock going backward in this manner,
and cause a crash, as reported in issue #869 (and in a long time ago in
commit 34232de9).

The fix is to to set _smp_init not when the first CPU's clock is initialized
(as we did until now), but only only after all CPUs' clocks have been set  
up -

at which point both system_time() and _boot_systemtime will be correct on
all CPUs.

The downside of this fix is that our clock continues to be stuck at time 0
for even longer during early boot - in my benchmark this time can grow to
almost 0.3 seconds for 32 vcpus. However, this "stuck time" does not cause
any real problems as code which needs functional clocks and preemptive
multitasking will run later anyway.

Signed-off-by: Nadav Har'El 
Message-Id: <20170418122956.7702-1-...@scylladb.com>
Reviewed-by: Benoît Canet 

---
diff --git a/drivers/clock-common.cc b/drivers/clock-common.cc
--- a/drivers/clock-common.cc
+++ b/drivers/clock-common.cc
@@ -9,6 +9,7 @@

 pv_based_clock::pv_based_clock()
 : _smp_init(false)
+, _boot_systemtime_init_counter(sched::cpus.size())
 , cpu_notifier([&] { setup_cpu(); })
 {
 }
@@ -17,10 +18,15 @@ void pv_based_clock::setup_cpu()
 {
 init_on_cpu();

-std::call_once(_boot_systemtime_init_flag, [&] {
+// We need to do the following once, after all CPUs ran their
+// init_init_on_cpu(), so any CPU calling uptime() will see not only
+// _boot_systemtime set, but also a functional system_time(). Until
+// all CPUs are set up, all of the will see zero uptime().
+if (_boot_systemtime_init_counter.fetch_sub(1,  
std::memory_order_relaxed)

+== 1) {
 _boot_systemtime = system_time();
 _smp_init.store(true, std::memory_order_release);
-});
+}
 }

 s64 pv_based_clock::time()
diff --git a/drivers/clock-common.hh b/drivers/clock-common.hh
--- a/drivers/clock-common.hh
+++ b/drivers/clock-common.hh
@@ -21,7 +21,7 @@ public:
 virtual s64 boot_time() override  
__attribute__((no_instrument_function));

 private:
 std::atomic _smp_init;
-std::once_flag _boot_systemtime_init_flag;
+std::atomic _boot_systemtime_init_counter;
 s64 _boot_systemtime;
 sched::cpu::notifier cpu_notifier;
 void setup_cpu();

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] command line: add makefile-like syntax to conditionally set environ var

[Commit Bot]

From: Justin Cinkelj 
Committer: Nadav Har'El 
Branch: master

command line: add makefile-like syntax to conditionally set environ var

Add "KEY?=value" syntax to set environ variable only if not set yet.
This allows runscript to contain default value in case KEY
is not set yet. The default value can be overriden by setting value
via REST API before runscript is started.

Fixes #870

Signed-off-by: Justin Cinkelj 
Message-Id: <20170421105050.21970-1-justin.cink...@xlab.si>

---
diff --git a/core/commands.cc b/core/commands.cc
--- a/core/commands.cc
+++ b/core/commands.cc
@@ -166,8 +166,22 @@ static void  
runscript_process_options(std::vector& re


 if (vars.count("env")) {
 for (auto t : vars["env"].as()) {
-debug("Setting in environment: %s\n", t);
-putenv(strdup(t.c_str()));
+size_t pos = t.find("?=");
+if (std::string::npos == pos) {
+// the basic "KEY=value" syntax
+debug("Setting in environment: %s\n", t);
+putenv(strdup(t.c_str()));
+}
+else {
+// "KEY?=value", makefile-like syntax, set variable  
only if not yet set

+auto key = t.substr(0, pos);
+auto value = t.substr(pos+2);
+if (nullptr == getenv(key.c_str())) {
+debug("Setting in environment: %s=%s\n", key,  
value);

+setenv(key.c_str(), value.c_str(), 1);
+}
+}
+
 }
 }

diff --git a/tests/tst-commands.cc b/tests/tst-commands.cc
--- a/tests/tst-commands.cc
+++ b/tests/tst-commands.cc
@@ -10,6 +10,7 @@

 #include 
 #include 
+#include 

 static int tests = 0, fails = 0;

@@ -760,6 +761,120 @@ static bool test_runscript_with_env_in_script()
 return true;
 }

+static bool test_runscript_with_conditional_env_in_script(bool  
set_env_vars_before_evaluation)

+{
+std::ofstream of1("/myscript", std::ios::out | std::ios::binary);
+of1 << "--env=ASDF?=ttrt --env=PORT?=$THEPORT /$THEPROG pp1a $PRM1b  
pp1c $PRM1d\n";

+of1.close();
+
+std::vector result;
+std::vector cmd = { "/prog1" };
+std::map expected_vars;
+size_t expected_size[] = {6};
+bool ok;
+
+// those two are set during command parsing
+if (NULL != getenv("ASDF")) {
+return false;
+}
+if (NULL != getenv("PORT")) {
+return false;
+}
+// those are required during command parsing
+if (0 != setenv("THEPORT", "4321", 1)) {
+return false;
+}
+if (0 != setenv("THEPROG", "prog1", 1)) {
+return false;
+}
+if (0 != setenv("PRM1b", "pp1b", 1)) {
+return false;
+}
+if (0 != setenv("PRM1d", "pp1d", 1)) {
+return false;
+}
+// run test with conditional variables set or clear
+if (set_env_vars_before_evaluation) {
+expected_vars["ASDF"] = "asdf-old";
+expected_vars["PORT"] = "port-old";
+if (0 != setenv("ASDF", expected_vars["ASDF"].c_str(), 1)) {
+return false;
+}
+if (0 != setenv("PORT", expected_vars["PORT"].c_str(), 1)) {
+return false;
+}
+}
+else {
+expected_vars["ASDF"] = "ttrt";
+expected_vars["PORT"] = "4321";
+}
+
+result = osv::parse_command_line(
+std::string("runscript \"/myscript\";  "),
+ok);
+
+if (!ok) {
+return false;
+}
+
+if (result.size() != 1) {
+return false;
+}
+
+for (size_t i = 0; i < result.size(); i++) {
+if (result[i].size() != expected_size[i]) {
+return false;
+}
+if (result[i][0] != cmd[i]) {
+return false;
+}
+}
+
+if (result[0][1] != std::string("pp1a")) {
+return false;
+}
+if (result[0][2] != std::string("pp1b")) {
+return false;
+}
+if (result[0][3] != std::string("pp1c")) {
+return false;
+}
+if (result[0][4] != std::string("pp1d")) {
+return false;
+}
+
+// environ variable with ? in name should not be created
+if (nullptr != getenv("ASDF?")) {
+return false;
+}
+if (nullptr != getenv("PORT?")) {
+return false;
+}
+// std::string(NULL) is undefined behavior, hence check getenv returns  
a valid string.

+if (nullptr == getenv("ASDF")) {
+return false;
+}
+if (nullptr == getenv("PORT")) {
+return false;
+}
+
+if (expected_vars["ASDF"] != getenv("ASDF")) {
+return false;
+}
+if (expected_vars["PORT"] != getenv("PORT")) {
+return false;
+}
+
+unsetenv("ASDF");
+unsetenv("PORT");
+unsetenv("THEPORT");
+unsetenv("THEPROG");
+unsetenv("PRM1b");
+ 

[COMMIT osv master] line-discipline: move and update comment

[Commit Bot]

From: Nadav Har'El 
Committer: Nadav Har'El 
Branch: master

line-discipline: move and update comment

The code in drivers/line-discipline.cc moved around a lot, and as a result
the comment trying to explain the purpose of this file got moved to the
middle of the file, and referred to no longer existing functions.

This patch moves and updates the comment. No actual code changes.

Signed-off-by: Nadav Har'El 
Message-Id: <20170313130650.20389-1-...@scylladb.com>

---
diff --git a/drivers/line-discipline.cc b/drivers/line-discipline.cc
--- a/drivers/line-discipline.cc
+++ b/drivers/line-discipline.cc
@@ -4,6 +4,33 @@
  * This work is open source software, licensed under the terms of the
  * BSD license as described in the LICENSE file in the top-level directory.
  */
+
+// Console line discipline thread.
+//
+// The "line discipline" is an intermediate layer between a byte-
+// oriented console driver (e.g., a serial port) and the Unix tty device
+// implementing features such as input echo, line editing, etc. In OSv,  
this

+// is implemented in a thread (see read_poll()), which is also responsible
+// for read-ahead (input characters are read, echoed and buffered even if
+// no-one is yet reading).
+//
+// The code below implements a fixed line discipline (actually two -  
canonical
+// and non-canonical). We resisted the temptation to make the line  
discipline

+// a stand-alone pluggable object: In the early 1980s, 8th Edition Research
+// Unix experimented with pluggable line disciplines, providing improved
+// editing features such as CRT erase (backspace outputs backspace-space-
+// backspace), word erase, etc. These pluggable line-disciplines led to the
+// development of Unix "STREAMS". However, today, these concepts are all  
but

+// considered obsolete: In the mid 80s it was realized that these editing
+// features can better be implemented in userspace code - Contemporary  
shells

+// introduced sophisticated command-line editing (tcsh and ksh were both
+// announced in 1983), and the line-editing libraries appeared (GNU  
Readline,

+// in 1989). Posix's standardization of termios(3) also more-or-less set in
+// stone the features that Posix-compliant line discipline should support.
+//
+// We currently support only a subset of the termios(3) features, which we
+// considered most useful. More of the features can be added as needed.
+
 #include "line-discipline.hh"
 #include 
 #include 
@@ -39,31 +66,6 @@ void LineDiscipline::read(struct uio *uio, int ioflag) {
 }
 }

-// Console line discipline thread.
-//
-// The "line discipline" is an intermediate layer between a physical device
-// (here a serial port) and a character-device interface (here  
console_read())
-// implementing features such as input echo, line editing, etc. In OSv,  
this
-// is implemented in a thread, which is also responsible for read-ahead  
(input

-// characters are read, echoed and buffered even if no-one is yet reading).
-//
-// The code below implements a fixed line discipline (actually two -  
canonical
-// and non-canonical). We resisted the temptation to make the line  
discipline

-// a stand-alone pluggable object: In the early 1980s, 8th Edition Research
-// Unix experimented with pluggable line disciplines, providing improved
-// editing features such as CRT erase (backspace outputs backspace-space-
-// backspace), word erase, etc. These pluggable line-disciplines led to the
-// development of Unix "STREAMS". However, today, these concepts are all  
but

-// considered obsolete: In the mid 80s it was realized that these editing
-// features can better be implemented in userspace code - Contemporary  
shells

-// introduced sophisticated command-line editing (tcsh and ksh were both
-// announced in 1983), and the line-editing libraries appeared (GNU  
Readline,

-// in 1989). Posix's standardization of termios(3) also more-or-less set in
-// stone the features that Posix-compliant line discipline should support.
-//
-// We currently support only a subset of the termios(3) features, which we
-// considered most useful. More of the features can be added as needed.
-
 static inline bool isctrl(char c) {
 return ((c<' ' && c!='\t' && c!='\n') || c=='\177');
 }

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] kill(): remove duplicate code

[Commit Bot]

From: Nadav Har'El 
Committer: Nadav Har'El 
Branch: master

kill(): remove duplicate code

Just a few lines above, we had exactly the same code... No need to do
it again.

Signed-off-by: Nadav Har'El 
Message-Id: <20170306120113.18737-1-...@scylladb.com>

---
diff --git a/libc/signal.cc b/libc/signal.cc
--- a/libc/signal.cc
+++ b/libc/signal.cc
@@ -353,10 +353,6 @@ int kill(pid_t pid, int sig)
 // FIXME: proper second (siginfo) and third (context)  
arguments (See example in call_signal_handler)

 sa.sa_sigaction(sig, nullptr, nullptr);
 } else {
-if (sa.sa_flags & SA_RESETHAND) {
-signal_actions[sig].sa_flags = 0;
-signal_actions[sig].sa_handler = SIG_DFL;
-}
 sa.sa_handler(sig);
 }
 },  
sched::thread::attr().detached().stack(65536).name("signal_handler"));


--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] scripts/setup.py: support for Fedora 25

[Commit Bot]

From: Nadav Har'El 
Committer: Nadav Har'El 
Branch: master

scripts/setup.py: support for Fedora 25

Signed-off-by: Nadav Har'El 

---
diff --git a/scripts/setup.py b/scripts/setup.py
--- a/scripts/setup.py
+++ b/scripts/setup.py
@@ -77,7 +77,14 @@ class Fedora_24(object):
 ec2_post_install = None
 version = '24'

-versions = [Fedora_19, Fedora_20, Fedora_21, Fedora_22, Fedora_23,  
Fedora_24]

+class Fedora_25(object):
+packages = ['java-1.8.0-openjdk', 'python2-requests']
+ec2_packages = []
+test_packages = []
+ec2_post_install = None
+version = '25'
+
+versions = [Fedora_19, Fedora_20, Fedora_21, Fedora_22, Fedora_23,  
Fedora_24, Fedora_25]


 class RHELbased(Fedora):
 name = ['Scientific Linux', 'NauLinux', 'CentOS Linux',
@@ -218,7 +225,7 @@ class Ubuntu_13_10(object):
 if dver.ec2_post_install:
 subprocess.check_call(dver.ec2_post_install,  
shell=True)

 sys.exit(0)
-print 'Your distribution version is not supported by this script'
+print ('Your distribution %s version %s is not supported by this  
script' % (name, version))

 sys.exit(1)

 print 'Your distribution is not supported by this script.'

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] tests/tst-kill: set flag global to 0 before waiting for 1

[Commit Bot]

From: Justin Cinkelj 
Committer: Nadav Har'El 
Branch: master

tests/tst-kill: set flag global to 0 before waiting for 1

The global was already set to 1 by kill above second wait loop.
If test VM/program was "slow" enough, the signal handler got cleaned
before "oldact.sa_handler == SIG_DFL" check (debug build, nested VM,
slow enough report printout). The test tended to fail when VM run on
physical host.

Fixes #861
Message-Id: <20170306122303.1251-1-justin.cink...@xlab.si>

---
diff --git a/tests/tst-kill.cc b/tests/tst-kill.cc
--- a/tests/tst-kill.cc
+++ b/tests/tst-kill.cc
@@ -130,6 +130,7 @@ int main(int ac, char** av)
 r = sigaction(SIGUSR1, nullptr, );
 report(r == 0 && oldact.sa_handler == handler1, "without SA_RESETHAND,  
signal handler is not reset");

 act.sa_flags = SA_RESETHAND;
+global = 0;
 r = sigaction(SIGUSR1, , nullptr);
 report(r == 0, "set SIGUSR1 handler with SA_RESETHAND");
 r = kill(0, SIGUSR1);

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] loader: fix log redirection append option

[Commit Bot]

From: Justin Cinkelj 
Committer: Nadav Har'El 
Branch: master

loader: fix log redirection append option

The O_APPEND flag was missing when "--redirect=>>my.log ..." was used.
Thus new output started to overwrite old content.

Signed-off-by: Justin Cinkelj 
Message-Id: <20170306143706.10745-1-justin.cink...@xlab.si>

---
diff --git a/loader.cc b/loader.cc
--- a/loader.cc
+++ b/loader.cc
@@ -447,7 +447,7 @@ void* do_main_thread(void *_main_args)
 bool append = (opt_redirect.substr(0, 2) == ">>");
 auto fn = opt_redirect.substr(append ? 2 : 0);
 int fd = open(fn.c_str(),
-O_WRONLY | O_CREAT | (append ? 0 : O_TRUNC), 777);
+O_WRONLY | O_CREAT | (append ? O_APPEND: O_TRUNC), 777);
 if (fd < 0) {
 perror("output redirection failed");
 } else {

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] tests/tst-kill: fix failing test on Linux

[Commit Bot]

From: Nadav Har'El 
Committer: Nadav Har'El 
Branch: master

tests/tst-kill: fix failing test on Linux

We tested that killing a non-existant process 17 resulted in an error.
But on Linux, it is quite possible that process 17 does exist, and this
test will fail. So change it to an even less likely to exist number.

Signed-off-by: Nadav Har'El 
Message-Id: <20170306134438.22129-1-...@scylladb.com>

---
diff --git a/tests/tst-kill.cc b/tests/tst-kill.cc
--- a/tests/tst-kill.cc
+++ b/tests/tst-kill.cc
@@ -49,7 +49,7 @@ int main(int ac, char** av)
 report(r == 0, "kill with signal 0 succeeds (and does nothing)");
 r = kill(-1, 0);
 report(r == 0, "kill of pid -1 is also fine");
-r = kill(17, 0);
+r = kill(17171717, 0);
 report(r == -1 && errno == ESRCH, "kill of non-existant process");
 r = kill(0, -2);
 report(r == -1 && errno == EINVAL, "kill with invalid signal number");

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] cli: fix run command

[Commit Bot]

From: Justin Cinkelj 
Committer: Nadav Har'El 
Branch: master

cli: fix run command

The c621a007340913aa4832f3fda195b29114a41a6a broke cli run command.
It was possible to "run --newprogram /myapp". But "run /myapp" didn't do
anything - it didn't even complain about "Failed to load object: /myapp".

Signed-off-by: Justin Cinkelj 
Message-Id: <20170802104652.22524-1-justin.cink...@xlab.si>

---
diff --git a/modules/cli/commands/run.lua b/modules/cli/commands/run.lua
--- a/modules/cli/commands/run.lua
+++ b/modules/cli/commands/run.lua
@@ -17,7 +17,8 @@ Options:

 cmd.main = function(args)
   local args, opts = cmd.parser:parse(args)
-  flag_newprogram = opts.newprogram
+  flag_newprogram = 0
+  if opts.newprogram then flag_newprogram=1 end

   for i = 1, #args do
 osv_request({"app"}, "PUT", {

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] runtime.c: avoid explicit "extern C" when header files are available

[Commit Bot]

From: Nadav Har'El 
Committer: Nadav Har'El 
Branch: master

runtime.c: avoid explicit "extern C" when header files are available

Signed-off-by: Nadav Har'El 

---
diff --git a/runtime.cc b/runtime.cc
--- a/runtime.cc
+++ b/runtime.cc
@@ -58,6 +58,8 @@
 #include 
 #include 
 #include 
+#include 
+#include 

 #define __LC_LAST 13

@@ -561,15 +563,13 @@ char *tmpnam_r(char *s)
 return s ? tmpnam(s) : NULL;
 }

-extern "C"
 pid_t wait3(int *status, int options, struct rusage *usage)
 {
 WARN_STUBBED();
 errno = ECHILD;
 return -1;
 }

-extern "C"
 pid_t wait4(pid_t pid, int *status, int options, struct rusage *usage)
 {
 WARN_STUBBED();
@@ -591,7 +591,6 @@ int getresgid(gid_t *rgid, gid_t *egid, gid_t *sgid)
 return -1;
 }

-extern "C"
 int openpty(int *amaster, int *aslave, char *name,
const struct termios *termp,
const struct winsize *winp)
@@ -601,7 +600,6 @@ int openpty(int *amaster, int *aslave, char *name,
 return -1;
 }

-extern "C"
 pid_t forkpty(int *amaster, char *name,
  const struct termios *termp,
  const struct winsize *winp)

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] libc: add functions required for libpython2.7.so

[Commit Bot]

From: Justin Cinkelj 
Committer: Nadav Har'El 
Branch: master

libc: add functions required for libpython2.7.so

When we try to reuse python from the build system (instead of
compiling it from source), libpython2.7.so requires additional
libc/glibc symbols. It seems that (at least for most trivial python
programs) those functions are not really called and can be implemented
as stubs. The patch was prepared on Fedora 25.

Signed-off-by: Justin Cinkelj 
Message-Id: <20170801161142.31048-1-justin.cink...@xlab.si>

---
diff --git a/Makefile b/Makefile
--- a/Makefile
+++ b/Makefile
@@ -1328,6 +1328,9 @@ libc += misc/mntent.o
 musl += misc/nftw.o
 libc += misc/__longjmp_chk.o

+musl += signal/killpg.o
+musl += signal/siginterrupt.o
+
 musl += multibyte/btowc.o
 musl += multibyte/internal.o
 musl += multibyte/mblen.o
@@ -1396,6 +1399,7 @@ libc += process/execle.o
 musl += process/execv.o
 musl += process/execl.o
 libc += process/waitpid.o
+musl += process/wait.o

 libc += arch/$(arch)/setjmp/setjmp.o
 libc += arch/$(arch)/setjmp/longjmp.o
@@ -1677,6 +1681,9 @@ libc += unistd/getsid.o
 libc += unistd/setsid.o
 libc += unistd/ttyname_r.o
 libc += unistd/ttyname.o
+musl += unistd/tcgetpgrp.o
+musl += unistd/tcsetpgrp.o
+musl += unistd/setpgrp.o

 musl += regex/fnmatch.o
 musl += regex/glob.o
diff --git a/runtime.cc b/runtime.cc
--- a/runtime.cc
+++ b/runtime.cc
@@ -554,3 +554,73 @@ int sysctl(int *, int, void *, size_t *, void *,  
size_t)

 errno = ENOTDIR;
 return -1;
 }
+
+extern "C"
+char *tmpnam_r(char *s)
+{
+return s ? tmpnam(s) : NULL;
+}
+
+extern "C"
+pid_t wait3(int *status, int options, struct rusage *usage)
+{
+WARN_STUBBED();
+errno = ECHILD;
+return -1;
+}
+
+extern "C"
+pid_t wait4(pid_t pid, int *status, int options, struct rusage *usage)
+{
+WARN_STUBBED();
+errno = ECHILD;
+return -1;
+}
+
+int getresuid(uid_t *ruid, uid_t *euid, uid_t *suid)
+{
+WARN_STUBBED();
+errno = ENOSYS;
+return -1;
+}
+
+int getresgid(gid_t *rgid, gid_t *egid, gid_t *sgid)
+{
+WARN_STUBBED();
+errno = ENOSYS;
+return -1;
+}
+
+extern "C"
+int openpty(int *amaster, int *aslave, char *name,
+   const struct termios *termp,
+   const struct winsize *winp)
+{
+WARN_STUBBED();
+errno = ENOENT;
+return -1;
+}
+
+extern "C"
+pid_t forkpty(int *amaster, char *name,
+ const struct termios *termp,
+ const struct winsize *winp)
+{
+WARN_STUBBED();
+errno = ENOENT;
+return -1;
+}
+
+int nice(int inc)
+{
+return setpriority(PRIO_PROCESS, 0, getpriority(PRIO_PROCESS, 0)+inc);
+}
+
+char *ctermid(char *s)
+{
+static char s2[L_ctermid];
+WARN_STUBBED();
+if (!s) s = s2;
+*s = 0;
+return s;
+}

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv-apps master] Add busybox

[Commit Bot]

From: Justin Cinkelj 
Committer: Nadav Har'El 
Branch: master

Add busybox

Busybox ash shell is more powerful than OSv --runscript option.
Use ash when fancy shell is required.

While busybox config file includes default options (e.g. nearly all
options are enabled), the usr.manifiest doesn't try to create all
valid symlinks to busybox binary. Instead, only a few are included.

A small test script is provided (commented out in usr.manifest).
It can be run as:
./scripts/run.py -e "--env=PATH=/usr/lib sleep 1.2; ash /cmd0.sh; echo Done"

Signed-off-by: Justin Cinkelj 
Message-Id: <20170807093251.14856-1-justin.cink...@xlab.si>

---
diff --git a/busybox/.gitignore b/busybox/.gitignore
--- a/busybox/.gitignore
+++ b/busybox/.gitignore
@@ -0,0 +1,3 @@
+build/
+usr.manifest
+
diff --git a/busybox/GET b/busybox/GET
--- a/busybox/GET
+++ b/busybox/GET
@@ -0,0 +1,40 @@
+#!/bin/bash
+
+set -e
+
+URL=https://github.com/mirror/busybox
+VERSION_GIT=1_27_stable
+VERSION_LIB=1.27.1
+
+[ ! -d build ] && mkdir build
+cd build
+if [ ! -d busybox ]
+then
+  git clone --branch $VERSION_GIT --depth 1 $URL
+  (cd busybox && git am ../../patches/000[0-9]-*\.patch)
+fi
+cd busybox
+
+cp -f ../../patches/config .config
+make -j4
+(cd 0_lib/ && ln -sf libbusybox.so.${VERSION_LIB} libbusybox.so)
+gcc -o busybox-main.so ../../patches/busybox-main.c -L0_lib -lbusybox  
-shared -fPIC

+cd ../..
+
+cat < usr.manifest
+/usr/lib/libbusybox.so.${VERSION_LIB}:  
\${MODULE_DIR}/build/busybox/0_lib/libbusybox.so.${VERSION_LIB}_unstripped

+/usr/lib/busybox: \${MODULE_DIR}/build/busybox/busybox-main.so
+/usr/lib/ash: ->/usr/lib/busybox
+#
+# common utilities
+/usr/lib/df: ->/usr/lib/busybox
+/usr/lib/sleep: ->/usr/lib/busybox
+/usr/lib/ls: ->/usr/lib/busybox
+/usr/lib/cat: ->/usr/lib/busybox
+/usr/lib/echo: ->/usr/lib/busybox
+/usr/lib/ping: ->/usr/lib/busybox
+#
+# test scripts
+## /**: \${MODULE_DIR}/test-script/**
+
+EOF
\ No newline at end of file
diff --git a/busybox/Makefile b/busybox/Makefile
--- a/busybox/Makefile
+++ b/busybox/Makefile
@@ -0,0 +1,13 @@
+#
+# Copyright (C) 2017 XLAB d.o.o.
+#
+# This work is open source software, licensed under the terms of the
+# BSD license as described in the LICENSE file in the top-level directory.
+#
+
+.PHONY: module
+module:
+   ./GET
+
+clean:
+   rm -rf build
diff --git a/busybox/patches/0001-OSv-remove-exit-usage.patch  
b/busybox/patches/0001-OSv-remove-exit-usage.patch

--- a/busybox/patches/0001-OSv-remove-exit-usage.patch
+++ b/busybox/patches/0001-OSv-remove-exit-usage.patch
@@ -0,0 +1,190 @@
+From 12453fd712c7874609eb4bcb2246a37c00571161 Mon Sep 17 00:00:00 2001
+From: Justin Cinkelj 
+Date: Thu, 3 Aug 2017 09:53:51 +0200
+Subject: [PATCH 1/3] OSv: remove exit() usage
+
+exit() would shutdown whole OSv VM, remove it.
+This requires removing noreturn attribute from related functions.
+
+Signed-off-by: Justin Cinkelj 
+---
+ include/libbb.h|  6 +++---
+ libbb/appletlib.c  | 28 ++--
+ libbb/verror_msg.c |  2 +-
+ libbb/xfunc_die.c  |  4 ++--
+ 4 files changed, 20 insertions(+), 20 deletions(-)
+
+diff --git a/include/libbb.h b/include/libbb.h
+index 8eccd81..bef7437 100644
+--- a/include/libbb.h
 b/include/libbb.h
+@@ -1108,7 +1108,7 @@ int spawn_and_wait(char **argv) FAST_FUNC;
+ int run_nofork_applet(int applet_no, char **argv) FAST_FUNC;
+ #ifndef BUILD_INDIVIDUAL
+ extern int find_applet_by_name(const char *name) FAST_FUNC;
+-extern void run_applet_no_and_exit(int a, const char *name, char **argv)  
NORETURN FAST_FUNC;
++extern int run_applet_no_and_exit(int a, const char *name, char **argv)  
/*NORETURN*/ FAST_FUNC;

+ #endif
+
+ /* Helpers for daemonization.
+@@ -1243,8 +1243,8 @@ extern smallint syslog_level;
+ extern smallint logmode;
+ extern uint8_t xfunc_error_retval;
+ extern void (*die_func)(void);
+-extern void xfunc_die(void) NORETURN FAST_FUNC;
+-extern void bb_show_usage(void) NORETURN FAST_FUNC;
++extern int xfunc_die(void) /*NORETURN*/ FAST_FUNC;
++extern int bb_show_usage(void) /*NORETURN*/ FAST_FUNC;
+ extern void bb_error_msg(const char *s, ...) __attribute__ ((format  
(printf, 1, 2))) FAST_FUNC;
+ extern void bb_error_msg_and_die(const char *s, ...) __attribute__  
((noreturn, format (printf, 1, 2))) FAST_FUNC;
+ extern void bb_perror_msg(const char *s, ...) __attribute__ ((format  
(printf, 1, 2))) FAST_FUNC;

+diff --git a/libbb/appletlib.c b/libbb/appletlib.c
+index df65849..c3e54a8 100644
+--- a/libbb/appletlib.c
 b/libbb/appletlib.c
+@@ -130,7 +130,7 @@ static const char *unpack_usage_messages(void)
+ #endif /* FEATURE_COMPRESS_USAGE */
+
+
+-void FAST_FUNC bb_show_usage(void)
++int FAST_FUNC bb_show_usage(void)
+ {
+   if (ENABLE_SHOW_USAGE) {
+ #ifdef SINGLE_APPLET_STR
+@@ -152,7 +152,7 @@ void FAST_FUNC bb_show_usage(void)
+   int ap = find_applet_by_name(applet_name);
+
+

[COMMIT osv master] Fixed httpserver module tests

[Commit Bot]

From: Waldemar Kozaczuk 
Committer: Nadav Har'El 
Branch: master

Fixed httpserver module tests

This patch fixes httpserver module tests by correcting 2 issues:
- tests/basetest.py was incorrectly loading jvm.json file
- the format of timestamp returned by os/date API call must have changed at  
some point to include 'UTC' and corresponding test was not adjusted


Signed-off-by: Waldemar Kozaczuk 
Message-Id: <1502301584-3838-1-git-send-email-jwkozac...@gmail.com>

---
diff --git a/modules/httpserver/tests/api/testos.py  
b/modules/httpserver/tests/api/testos.py

--- a/modules/httpserver/tests/api/testos.py
+++ b/modules/httpserver/tests/api/testos.py
@@ -19,7 +19,7 @@ def test_os_uptime(self):
 def test_os_date(self):
 path = self.path_by_nick(self.os_api, "os_date")
 val = self.curl(path).encode('ascii', 'ignore')
- 
self.assertRegexpMatches(val, "...\\s+...\\s+\\d+\\s+\\d\\d:\\d\\d:\\d\\d\\s+20..",  
path)
+ 
self.assertRegexpMatches(val, "...\\s+...\\s+\\d+\\s+\\d\\d:\\d\\d:\\d\\d\\s+UTC\\s+20..",  
path)


 def test_os_total_memory(self):
 path = self.path_by_nick(self.os_api, "os_memory_total")
diff --git a/modules/httpserver/tests/basetest.py  
b/modules/httpserver/tests/basetest.py

--- a/modules/httpserver/tests/basetest.py
+++ b/modules/httpserver/tests/basetest.py
@@ -25,7 +25,11 @@ def get_url(cls, uri):

 @classmethod
 def get_json_api(cls, name):
-json_data = open(os.path.join(cls.config.jsondir, name))
+return cls.get_json_api_from_directory(cls.config.jsondir,name)
+
+@classmethod
+def get_json_api_from_directory(cls, directory, name):
+   json_data = open(os.path.join(directory, name))
 data = json.load(json_data)
 json_data.close()
 return data
@@ -145,7 +149,8 @@ def shutdown(cls):

 @classmethod
 def start_image(cls):
-cls.jvm_api = cls.get_json_api("jvm.json")
+jvm_plugin_api_listings_path =  
os.path.join(os.path.realpath(os.path.dirname(__file__)),'../../httpserver-jvm-plugin/api-doc/listings')
+cls.jvm_api =  
cls.get_json_api_from_directory(jvm_plugin_api_listings_path,"jvm.json")

 cls.os_api = cls.get_json_api("os.json")
 if not cls.config.connect:
 cls.os_process = cls.exec_os()

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] command line: expand env vars before using them to expand cmdline

[Commit Bot]

From: Justin Cinkelj 
Committer: Nadav Har'El 
Branch: master

command line: expand env vars before using them to expand cmdline

Previously, runscript with content
"--env=PORT?= /usr/lib/mpi_hello.so aaa $PORT ccc",
run as
/scripts/run.py -Vd -e '--env=PORT=newport runscript /script/my-script'
started mpi_hello.so with
argv[2] == "newport" and environ["PORT"] == "".
This was wrong and confusing.

Fix it by first evaluating env var, and only then insert env var values
into final command line.

Signed-off-by: Justin Cinkelj 
Message-Id: <20170711143127.6901-1-justin.cink...@xlab.si>

---
diff --git a/core/commands.cc b/core/commands.cc
--- a/core/commands.cc
+++ b/core/commands.cc
@@ -96,23 +96,31 @@ Everything after first $ is assumed to be env var.
 So with AA=aa, "$AA" -> "aa", and "X$AA" => "Xaa"
 More than one $ is not supported, and "${AA}" is also not.
 */
+void expand_environ_vars(std::string& word)
+{
+size_t pos;
+if ((pos = word.find_first_of('$')) != std::string::npos) {
+std::string new_word = word.substr(0, pos);
+std::string key = word.substr(pos+1);
+auto tmp = getenv(key.c_str());
+//debug("new_word=%s, key=%s, tmp=%s\n", new_word.c_str(),  
key.c_str(), tmp);

+if (tmp) {
+new_word += tmp;
+}
+word = new_word;
+}
+}
+
+/*
+Expand environ vars in each word.
+*/
 void expand_environ_vars(std::vector& result)
 {
 std::vector::iterator cmd_iter;
 std::vector::iterator line_iter;
 for (cmd_iter=result.begin(); cmd_iter!=result.end(); cmd_iter++) {
 for (line_iter=cmd_iter->begin(); line_iter!=cmd_iter->end();  
line_iter++) {

-size_t pos;
-if ((pos = line_iter->find_first_of('$')) !=  
std::string::npos) {

-std::string new_word = line_iter->substr(0, pos);
-std::string key = line_iter->substr(pos+1);
-auto tmp = getenv(key.c_str());
-//debug("new_word=%s, key=%s, tmp=%s\n",  
new_word.c_str(), key.c_str(), tmp);

-if (tmp) {
-new_word += tmp;
-}
-*line_iter = new_word;
-}
+expand_environ_vars(*line_iter);
 }
 }
 }
@@ -167,21 +175,36 @@ static void  
runscript_process_options(std::vector& re

 if (vars.count("env")) {
 for (auto t : vars["env"].as()) {
 size_t pos = t.find("?=");
+std::string key, value;
 if (std::string::npos == pos) {
 // the basic "KEY=value" syntax
-debug("Setting in environment: %s\n", t);
-putenv(strdup(t.c_str()));
+size_t pos2 = t.find("=");
+assert(std::string::npos != pos2);
+key = t.substr(0, pos2);
+value = t.substr(pos2+1);
+//debug("Setting in environment (def): %s=%s\n", key,  
value);

 }
 else {
 // "KEY?=value", makefile-like syntax, set variable  
only if not yet set

-auto key = t.substr(0, pos);
-auto value = t.substr(pos+2);
-if (nullptr == getenv(key.c_str())) {
-debug("Setting in environment: %s=%s\n", key,  
value);

-setenv(key.c_str(), value.c_str(), 1);
+key = t.substr(0, pos);
+value = t.substr(pos+2);
+if (nullptr != getenv(key.c_str())) {
+// key already used, do not overwrite it
+//debug("NOT setting in environment  
(makefile): %s=%s\n", key, value);

+key = "";
+}
+else {
+//debug("Setting in environment  
(makefile): %s=%s\n", key, value);

 }
 }

+if (key.length() > 0) {
+// we have something to set
+expand_environ_vars(value);
+debug("Setting in environment: %s=%s\n", key, value);
+setenv(key.c_str(), value.c_str(), 1);
+}
+
 }
 }

@@ -191,7 +214,7 @@ static void  
runscript_process_options(std::vector& re

 }

 /*
-If cmd starts with "runcript file", read content of file and
+If cmd starts with "runscript file", read content of file and
 return vector of all programs to be run.
 File can contain multiple commands per line.
 ok flag is set to false on parse error, and left unchanged otherwise.
@@ -234,10 +257,13 @@ runscript_expand(const std::vector& cmd,  
bool , bool _runscri

 return result2;
 }
 // Replace env vars found 

[COMMIT osv master] add openat64() and creat64() functions

[Commit Bot]

From: Nadav Har'El 
Committer: Nadav Har'El 
Branch: master

add openat64() and creat64() functions

Add functions openat64() and creat64() as aliases to the usual functions.

Refs #747.

Signed-off-by: Nadav Har'El 
Message-Id: <20170712062906.2401-1-...@scylladb.com>

---
diff --git a/fs/vfs/main.cc b/fs/vfs/main.cc
--- a/fs/vfs/main.cc
+++ b/fs/vfs/main.cc
@@ -197,6 +197,7 @@ int openat(int dirfd, const char *pathname, int  
flags, ...)


 return error;
 }
+LFS64(openat);

 // open() has an optional third argument, "mode", which is only needed in
 // some cases (when the O_CREAT mode is used). As a safety feature, recent
@@ -222,6 +223,7 @@ int creat(const char *pathname, mode_t mode)
 {
 return open(pathname, O_CREAT|O_WRONLY|O_TRUNC, mode);
 }
+LFS64(creat);

 TRACEPOINT(trace_vfs_close, "%d", int);
 TRACEPOINT(trace_vfs_close_ret, "");
diff --git a/include/api/fcntl.h b/include/api/fcntl.h
--- a/include/api/fcntl.h
+++ b/include/api/fcntl.h
@@ -159,9 +159,6 @@ ssize_t tee(int, int, size_t, unsigned);
 #endif

 #if defined(_LARGEFILE64_SOURCE) || defined(_GNU_SOURCE)
-//#define open64 open
-#define openat64 openat
-#define creat64 creat
 #define off64_t off_t
 #endif

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] Support mmap() flag MAP_32BIT

[Commit Bot]

From: Nadav Har'El 
Committer: Nadav Har'El 
Branch: master

Support mmap() flag MAP_32BIT

Support the mmap() flag MAP_32BIT which asks for the allocated address
to be in the lower 32 bits. Some applications (like wrk and luajit)
use this flag and need it to work as expected.

By default, OSv mappings look for a hole starting at 0x2000.
With this patch and MAP_32BIT, we change that to 0x200.

Signed-off-by: Nadav Har'El 
Message-Id: <20170711145137.28618-1-...@scylladb.com>

---
diff --git a/libc/mman.cc b/libc/mman.cc
--- a/libc/mman.cc
+++ b/libc/mman.cc
@@ -130,6 +130,13 @@ void *mmap(void *addr, size_t length, int prot, int  
flags,

 auto mmap_flags = libc_flags_to_mmap(flags);
 auto mmap_perm  = libc_prot_to_perm(prot);

+if ((flags & MAP_32BIT) && !(flags & MAP_FIXED) && !addr) {
+// If addr is not specified, OSv by default starts mappings at  
address
+// 0x2000ul (see mmu::allocate()).  MAP_32BIT asks for a  
lower
+// default. If MAP_FIXED or addr were specified, the default does  
not

+// matter anyway.
+addr = (void*)0x200ul;
+}
 if (flags & MAP_ANONYMOUS) {
 // We have already determined (see below) the region where the  
heap must be located. Now the JVM will request

 // fixed mappings inside that region

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] tests: avoid GOLD linker on three problematic tests

[Commit Bot]

From: Nadav Har'El 
Committer: Nadav Har'El 
Branch: master

tests: avoid GOLD linker on three problematic tests

Three tests fail if the system's default linker is gold. In OSv's makefile,
we explicitly choose the bfd linker because of gold's shortcomings, but in
the test Makefile we let the build machine's distribution to choose its own
default.

Fixes failures on tst-mmap, tst-elf-permissions and tst-tls.
Fixes #891 (this is more a work-around than a fix, but it will do for now).

Signed-off-by: Nadav Har'El 

---
diff --git a/modules/tests/Makefile b/modules/tests/Makefile
--- a/modules/tests/Makefile
+++ b/modules/tests/Makefile
@@ -97,6 +97,14 @@ tests += testrunner.so
 $(out)/tests/tst-mmap.so: COMMON += -Wl,-z,now
 $(out)/tests/tst-elf-permissions.so: COMMON += -Wl,-z,relro

+# The following tests use special linker trickery which apprarently
+# doesn't work as expected with GOLD linker, so we need to choose BFD.
+# TODO: figure out why this workaround was needed (the reason may be
+# different for each of these tests), and avoid this workaround!
+$(out)/tests/tst-mmap.so: COMMON += -fuse-ld=bfd
+$(out)/tests/tst-elf-permissions.so: COMMON += -fuse-ld=bfd
+$(out)/tests/tst-tls.so: COMMON += -fuse-ld=bfd
+
 $(out)/tests/tst-tls.so: \
$(src)/tests/tst-tls.cc \
$(out)/tests/libtls.so

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] boot.S: Increase main() stack size

[Commit Bot]

From: Nadav Har'El 
Committer: Nadav Har'El 
Branch: master

boot.S: Increase main() stack size

We use "init_stack_top" as the end of the stack for loader.cc's premain()
and main() functions (all other threads will allocate their stacks
dynamically). A size of 4*4096 = 16384 bytes was reserved for this stack,
but we actually use more (main() puts a sched::thread object on the stack
and its size is currently 20880 bytes.

So this patch increase this boot stack size to 10*4096 = 40960 bytes.
Note that as currently written, this stack continues to take up space even
after boot, so we don't want it excessively big.

I'm not sure how this overflow didn't cause any serious problems. What we
have right before this stack is the percpu section, and it's quite possible
that when most parts of it (e.g., worker queues, trace buffers, etc.) are
overwritten in early boot nothing bad happens because nothing is using
these areas at that time. But it's certainly not a good reason to leave
this bug behind. Especially not when we know of bugs which one guess is
that they are caused by overwrite of the per-cpu area (see issue #382).

Signed-off-by: Nadav Har'El 
Message-Id: <20170712211813.18741-1-...@scylladb.com>

---
diff --git a/arch/x64/boot.S b/arch/x64/boot.S
--- a/arch/x64/boot.S
+++ b/arch/x64/boot.S
@@ -44,7 +44,7 @@ gdt_end = .
 .bss

 .align 16
-. = . + 4096*4
+. = . + 4096*10
 init_stack_top = .

 . = . + 4096*10

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] gcc 7: fix alignment of main()'s stack

[Commit Bot]

From: Nadav Har'El 
Committer: Nadav Har'El 
Branch: master

gcc 7: fix alignment of main()'s stack

This patch fixes an early boot crash (a #GP) of the optimized build
built with gcc 7.

According to the x86_64 ABI, C functions must be called with the stack
16-byte aligned. The called function may and do use this assumption to
create aligned objects on the stack without wasting time to verify the
actual alignment of the stack. We've gone through great lengths to ensure
this alignment guarantee is preserved throughout OSv, but one place we
got it wrong: where we call loader.cc's main() function, we set up a
properly aligned stack, but the used JMP instead of CALL to call the
function main(). It might seem that using jmp to a function that will
never return anyway is fine. But it is not fine, because CALL pushes an
8-byte return address on the stack, which means that JMP leaves the
stack offset by 8 bytes from the expected 16-byte alignment.

An interesting question is why we only noticed this now, with gcc 7, when
this is one of the oldest lines of codes in OSv (from December 2012).
The reason is that this bug only effects loader.cc's main() function, and
no other function or thread in OSv, and main() is a tiny function which
does very little and has little chance to care about alignment. However,
one thing it does do is to create a "thread" object on the stack.
The compiler thinks this thread object is 16-byte aligned
(alignof(thread)==16) but it is not, it is offset by 8 bytes from this
correct alignment. It appears that with gcc 7's optimizer, something in
thread's constructor uses certain instructions which only work on 16-byte
aligned data (e.g., MOVAPS), which the compiler assumes, and when we are
wrongly aligned the result is a general protection fault and a crash.
Without optimizations enabled, this wrong alignment doesn't cause this
crash, even on gcc 7.

The patch trivially fixes JMP to CALL, which is what the callee's (main())
expects we would call to preserve the stack's alignment.

Signed-off-by: Nadav Har'El 
Message-Id: <20170712185922.16302-1-...@scylladb.com>

---
diff --git a/arch/x64/boot.S b/arch/x64/boot.S
--- a/arch/x64/boot.S
+++ b/arch/x64/boot.S
@@ -108,7 +108,7 @@ start64:
 call premain
 mov __argc, %edi
 mov __argv, %rsi
-jmp main
+call main
 .cfi_endproc

 # The smp trampoline must be in the lower 1MB, so we manually relocate

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] gcc 7: do not use dynamic exception specifications

[Commit Bot]

From: Nadav Har'El 
Committer: Nadav Har'El 
Branch: master

gcc 7: do not use dynamic exception specifications

Dynamic exception specifications (throw(...) specification on functions)
were deprecated in C++11 so we shouldn't use them. Gcc 7 started complaining
about them, so it's about time we get rid of them.

Use the C++11 "noexcept" instead, where appropriate (instead of an empty
throw()).

Signed-off-by: Nadav Har'El 
Message-Id: <20170712151334.12673-5-...@scylladb.com>

---
diff --git a/include/osv/sched.hh b/include/osv/sched.hh
--- a/include/osv/sched.hh
+++ b/include/osv/sched.hh
@@ -1009,9 +1009,9 @@ void preempt_enable();
 class interruptible
 {
 public:
-static void prepare(thread* target_thread) throw()
+static void prepare(thread* target_thread) noexcept
 { target_thread->interrupted(false); }
-static void check(thread* target_thread, wait_guard& wg) throw(int) {
+static void check(thread* target_thread, wait_guard& wg) {
 if(target_thread->interrupted()) {
 wg.stop();
 throw int(EINTR);
@@ -1022,9 +1022,9 @@ public:
 class noninterruptible
 {
 public:
-static void prepare(thread* target_thread) throw()
+static void prepare(thread* target_thread) noexcept
 {}
-static void check(thread* target_thread, wait_guard& wg) throw()
+static void check(thread* target_thread, wait_guard& wg) noexcept
 {}
 };

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] Remove unused interrupt stack

[Commit Bot]

From: Nadav Har'El 
Committer: Nadav Har'El 
Branch: master

Remove unused interrupt stack

In boot.S we reserved 40960 bytes for an "interrupt stack" that is not
actually used anywhere since almost the start of OSv (see commit
da6ba16dbd182517e4c889f806cd9ce57151). So get rid of it.

Signed-off-by: Nadav Har'El 
Message-Id: <20170712211813.18741-2-...@scylladb.com>

---
diff --git a/arch/x64/boot.S b/arch/x64/boot.S
--- a/arch/x64/boot.S
+++ b/arch/x64/boot.S
@@ -47,10 +47,6 @@ gdt_end = .
 . = . + 4096*10
 init_stack_top = .

-. = . + 4096*10
-.global interrupt_stack_top
-interrupt_stack_top = .
-
 .text

 #define BOOT_CR0 ( X86_CR0_PE \

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] cloud-init: support standard ISO images

[Commit Bot]

From: Justin Cinkelj 
Committer: Nadav Har'El 
Branch: master

cloud-init: support standard ISO images

In 56c6195b252b127c7fe90cb9fa46c370b384b51a was added option for
reading configuration file from a secondary disk. This is so-called
NoCloud configuration method, see also
http://cloudinit.readthedocs.io/en/0.7.9/topics/datasources/nocloud.html.
Due to missing ISO filesystem support, a custom disk format was used.
This patch replaces custom disk format with standard ISO images.
ISO FS support is provided by libcdio.

After preparing user-data file (meta-data file is currently ignored),
the image can be build with cloud-localds program (provided by cloud-utils
package on fedora).

Signed-off-by: Justin Cinkelj 
Message-Id: <20170712213724.4347-1-justin.cink...@xlab.si>

---
diff --git a/modules/cloud-init/Makefile b/modules/cloud-init/Makefile
--- a/modules/cloud-init/Makefile
+++ b/modules/cloud-init/Makefile
@@ -1,4 +1,5 @@
-INCLUDES += -I../../include -I.
+INCLUDES = -I. -I../../include -I../../arch/$(ARCH) -I../.. \
+-I../../build/$(mode)/gen/include

 autodepend = -MD -MT $@ -MP
 CXXFLAGS  = -g -rdynamic -Wall -std=c++11 -fPIC $(INCLUDES) $(autodepend)
diff --git a/modules/cloud-init/main.cc b/modules/cloud-init/main.cc
--- a/modules/cloud-init/main.cc
+++ b/modules/cloud-init/main.cc
@@ -5,6 +5,9 @@
  * BSD license as described in the LICENSE file in the top-level directory.
  */

+#include 
+#include 
+#include 
 #include "cloud-init.hh"
 #include "files-module.hh"
 #include "server-module.hh"
@@ -15,59 +18,60 @@
 #include 
 #include 
 #include 
+#include 

 using namespace std;
 using namespace init;
 namespace po = boost::program_options;

-// config_disk() checks whether we have a second disk (/dev/vblk1) which
-// holds nothing but a configuration file, and if there is, it copies the
-// configuration to the given file.
-// Currently, the configuration disk must be in a trivial format generated
-// by scripts/file2img: A magic header, then the length of the file (in
-// decimal), followed by the content.
+// config_disk() allows to use NoCloud VM configuration method - see
+//  
http://cloudinit.readthedocs.io/en/0.7.9/topics/datasources/nocloud.html.

+// NoCloud method provides two files with cnfiguration data (/user-data and
+// /meta-data) on a disk. The disk is required to have label "cidata".
+// It can contain ISO9660 or FAT filesystem.
+//
+// config_disk() checks whether we have a second disk (/dev/vblk1) with
+// ISO image, and if there is, it copies the configuration file from
+// /user-data to the given file.
 // config_disk() returns true if it has successfully read the configuration
 // into the requested file.
+//
+// OSv implementation limitations:
+// The /meta-data file is currently ignored.
+// Only ISO9660 filesystem is supported.
+// The mandatory "cidata" volume label is not checked.
+//
+// Example ISO image can be created by running
+// cloud-localds cloud-init.img cloud-init.yaml
+// The cloud-localds command is provided by cloud-utils package (fedora).
 static bool config_disk(const char* outfile) {
-int fd = open("/dev/vblk1", O_RDONLY);
-if (fd < 0) {
+char disk[] = "/dev/vblk1";
+char srcfile[] = "/user-data";
+struct stat sb;
+int ret;
+int app_ret = -1;
+
+ret = stat(disk, );
+if (ret != 0) {
 return false;
 }
-char data[512];
-ssize_t r = read(fd, data, sizeof(data));
-static const char* magic = "!file_in_image\n";
-ssize_t magic_len = strlen(magic);
-if (r < magic_len || strncmp(data, magic, magic_len)) {
-close(fd);
+
+std::vector cmd = {"/usr/bin/iso-read.so", "-e",  
srcfile, "-o", outfile, disk};

+osv::run(cmd[0], cmd, _ret);
+if (app_ret != 0) {
+debug("cloud-init: warning, %s exited with code %d (%s is not ISO  
image?)\n", cmd[0], app_ret, disk);

 return false;
 }
-debug("cloud-init: found configuration in /dev/vblk1\n");
-int out = open(outfile, O_WRONLY|O_CREAT, 0777);
-if (out < 0) {
-debug("cloud-init: failed to copy configuration to %s\n", outfile);
-close(fd);
+ret = stat(outfile, );
+if (ret != 0) {
+debug("cloud-init: stat(%s) failed, errno=%d\n", outfile, errno);
 return false;
 }
-unsigned offset = magic_len;
-while (offset < r && data[offset] != '\n')
-offset++;
-offset++; // skip the \n too
-ssize_t filelen = atoi(data + magic_len);
-while (filelen) {
-int len = std::min(filelen, r - offset);
-if (len <= 0) {
-debug("cloud-init: unexpected end of image\n", outfile);
-close(fd);
-close(out);
-return false;
-}
-write(out, data + offset, len);
-filelen -= len;
-offset = 0;
-r = read(fd, data, sizeof(data));
+if ((sb.st_mode & S_IFMT) != S_IFREG) {
+debug("cloud-init: 

[COMMIT osv master] Added ability to OSv boot logic to detect if running of Hyper/V and implemented Hyper/V specific clock.

[Commit Bot]

From: Waldemar Kozaczuk 
Committer: Nadav Har'El 
Branch: master

Added ability to OSv boot logic to detect if running of Hyper/V and  
implemented Hyper/V specific clock.


These changes are enough to allow execution of simple OSv apps that do NOT  
need
networking nor storage access. The image built using the command below  
should

successfully boot and execute on Hyper/V:

scripts/build image=native-example fs=ramfs

Here is the list of specific changes:
 - added hyperv_clocksource to features_type to indicate that Hyper/V  
reference time count MSR is available
 - modified cpuid.cc to alllow detection availability of Hyper/V reference  
time count MSR
 - moved rtc class from hpet file into separate header and implementation  
file so that it can be shared between hpet and hy

pervclock
 - implemented hypervclock class to provide time source when running on  
Hyper/V


Fixes #874

Signed-off-by: Waldemar Kozaczuk 
Message-Id: <1499985434-3665-1-git-send-email-jwkozac...@gmail.com>

---
diff --git a/Makefile b/Makefile
--- a/Makefile
+++ b/Makefile
@@ -820,9 +820,10 @@ drivers += drivers/vmxnet3-queues.o
 drivers += drivers/virtio-blk.o
 drivers += drivers/virtio-scsi.o
 drivers += drivers/virtio-rng.o
-drivers += drivers/kvmclock.o drivers/xenclock.o
+drivers += drivers/kvmclock.o drivers/xenclock.o drivers/hypervclock.o
 drivers += drivers/acpi.o
 drivers += drivers/hpet.o
+drivers += drivers/rtc.o
 drivers += drivers/xenfront.o drivers/xenfront-xenbus.o  
drivers/xenfront-blk.o

 drivers += drivers/pvpanic.o
 drivers += drivers/ahci.o
diff --git a/arch/x64/cpuid.cc b/arch/x64/cpuid.cc
--- a/arch/x64/cpuid.cc
+++ b/arch/x64/cpuid.cc
@@ -8,6 +8,7 @@
 #include "cpuid.hh"
 #include "processor.hh"
 #include "xen.hh"
+#include 

 namespace processor {

@@ -109,12 +110,19 @@ void process_xen_bits(features_type )
 }
 }

+void process_hyperv_bits(features_type ) {
+if(hyperv_identify() && hyperv_is_timecount_available()) {
+features.hyperv_clocksource = true;
+}
+}
+
 void process_cpuid(features_type& features)
 {
 for (unsigned i = 0; i < nr_cpuid_bits; ++i) {
 process_cpuid_bit(features, cpuid_bits[i]);
 }
 process_xen_bits(features);
+process_hyperv_bits(features);
 }

 }
diff --git a/arch/x64/cpuid.hh b/arch/x64/cpuid.hh
--- a/arch/x64/cpuid.hh
+++ b/arch/x64/cpuid.hh
@@ -37,6 +37,7 @@ struct features_type {
 bool xen_clocksource;
 bool xen_vector_callback;
 bool xen_pci;
+bool hyperv_clocksource;
 };

 extern const features_type& features();
diff --git a/bsd/sys/dev/hyperv/include/hyperv.h  
b/bsd/sys/dev/hyperv/include/hyperv.h

--- a/bsd/sys/dev/hyperv/include/hyperv.h
+++ b/bsd/sys/dev/hyperv/include/hyperv.h
@@ -31,6 +31,8 @@
 #ifndef _HYPERV_H_
 #define _HYPERV_H_

+#include 
+
 #ifdef _KERNEL

 #include 
@@ -93,6 +95,7 @@ extern u_inthyperv_features;/*  
CPUID_HV_MSR_ */


 boolhyperv_identify();
 boolhyperv_is_timecount_available();
+uint64_thyperv_tc64_rdmsr();

 #endif  /* _KERNEL */

diff --git a/bsd/sys/dev/hyperv/vmbus/hyperv.cc  
b/bsd/sys/dev/hyperv/vmbus/hyperv.cc

--- a/bsd/sys/dev/hyperv/vmbus/hyperv.cc
+++ b/bsd/sys/dev/hyperv/vmbus/hyperv.cc
@@ -32,8 +32,6 @@
 #include 
 __FBSDID("$FreeBSD$");

-#include 
-
 #include 
 #include 
 #include 
diff --git a/drivers/hpet.cc b/drivers/hpet.cc
--- a/drivers/hpet.cc
+++ b/drivers/hpet.cc
@@ -9,7 +9,6 @@ extern "C" {
 #include "acpi.h"
 }
 #include 
-#include 
 #include 
 #include "processor.hh"
 #include "clock.hh"
@@ -18,6 +17,7 @@ extern "C" {
 #include "arch.hh"
 #include 
 #include 
+#include "rtc.hh"

 using boost::intrusive::get_parent_from_member;

@@ -33,71 +33,6 @@ class hpetclock : public clock {
 uint64_t _period;
 };

-class rtc {
-public:
-rtc();
-uint64_t wallclock_ns();
-private:
-bool _is_bcd;
-uint8_t cmos_read(uint8_t val);
-uint8_t cmos_read_date(uint8_t val);
-};
-
-
-#define RTC_PORT(x) (0x70 + (x))
-#define RTC_BINARY_DATE 0x4
-
-rtc::rtc()
-{
-auto status = cmos_read(0xB);
-_is_bcd = !(status & RTC_BINARY_DATE);
-}
-
-uint8_t rtc::cmos_read(uint8_t addr)
-{
-processor::outb(addr, RTC_PORT(0));
-return processor::inb(RTC_PORT(1));
-}
-
-uint8_t rtc::cmos_read_date(uint8_t addr)
-{
-uint8_t val = cmos_read(addr);
-if (!_is_bcd)
-return val;
-return (val & 0x0f) + (val >> 4) * 10;
-}
-
-uint64_t rtc::wallclock_ns()
-{
-// 0x80 : Update in progress. Wait for it.
-while ((cmos_read(0xA) & 0x80));
-
-uint8_t year = cmos_read_date(9);
-uint8_t month = cmos_read_date(8);
-uint8_t day = cmos_read_date(7);
-uint8_t hours = cmos_read_date(4);
-uint8_t mins = cmos_read_date(2);
-uint8_t secs = cmos_read_date(0);
-
-// FIXME: Get century from FADT.
-auto gdate = boost::gregorian::date(2000 + year, month, day);
-
-// My understanding 

[COMMIT osv master] Added subset of original FreeBSD driver code for Hyper/V.

[Commit Bot]

From: Waldemar Kozaczuk 
Committer: Nadav Har'El 
Branch: master

Added subset of original FreeBSD driver code for Hyper/V.

- added subset of original FreeBSD driver code for Hyper/V
  
(https://github.com/freebsd/freebsd/tree/4f55a572a1ac518b3e4f5a856ec0b403fe23ed5e)
 for now limited to VMBus driver basic functionality and common header  
files.

- renamed all *.c files to *.cc.
- replaced tabs with spaces.

Most of this code is not necessary to make OSv boot on Hyper/V however it  
is included in anticipation of the network/disk support later.


Signed-off-by: Waldemar Kozaczuk 
Message-Id: <1499985397-3562-1-git-send-email-jwkozac...@gmail.com>

---
diff --git a/bsd/sys/dev/hyperv/include/hyperv.h  
b/bsd/sys/dev/hyperv/include/hyperv.h

--- a/bsd/sys/dev/hyperv/include/hyperv.h
+++ b/bsd/sys/dev/hyperv/include/hyperv.h
@@ -0,0 +1,96 @@
+/*-
+ * Copyright (c) 2009-2012,2016 Microsoft Corp.
+ * Copyright (c) 2012 NetApp Inc.
+ * Copyright (c) 2012 Citrix Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *notice unmodified, this list of conditions, and the following
+ *disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *notice, this list of conditions and the following disclaimer in the
+ *documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED  
WARRANTIES

+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF  
USE,

+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _HYPERV_H_
+#define _HYPERV_H_
+
+#ifdef _KERNEL
+
+#include 
+#include 
+
+#define MSR_HV_TIME_REF_COUNT   0x4020
+
+#define CPUID_HV_MSR_TIME_REFCNT0x0002  /* MSR_HV_TIME_REF_COUNT */
+#define CPUID_HV_MSR_SYNIC  0x0004  /* MSRs for SynIC */
+#define CPUID_HV_MSR_SYNTIMER   0x0008  /* MSRs for SynTimer */
+#define CPUID_HV_MSR_APIC   0x0010  /* MSR_HV_{EOI,ICR,TPR} */
+#define CPUID_HV_MSR_HYPERCALL  0x0020  /* MSR_HV_GUEST_OS_ID
+ * MSR_HV_HYPERCALL */
+#define CPUID_HV_MSR_VP_INDEX   0x0040  /* MSR_HV_VP_INDEX */
+#define CPUID_HV_MSR_REFERENCE_TSC  0x0200  /* MSR_HV_REFERENCE_TSC */
+#define CPUID_HV_MSR_GUEST_IDLE 0x0400  /* MSR_HV_GUEST_IDLE */
+
+#ifndef NANOSEC
+#define NANOSEC 10ULL
+#endif
+#define HYPERV_TIMER_NS_FACTOR  100ULL
+#define HYPERV_TIMER_FREQ   (NANOSEC / HYPERV_TIMER_NS_FACTOR)
+
+#endif  /* _KERNEL */
+
+#define HYPERV_REFTSC_DEVNAME   "hv_tsc"
+
+/*
+ * Hyper-V Reference TSC
+ */
+struct hyperv_reftsc {
+volatile uint32_t   tsc_seq;
+volatile uint32_t   tsc_rsvd1;
+volatile uint64_t   tsc_scale;
+volatile int64_ttsc_ofs;
+} __packed __aligned(PAGE_SIZE);
+#ifdef CTASSERT
+CTASSERT(sizeof(struct hyperv_reftsc) == PAGE_SIZE);
+#endif
+
+#ifdef _KERNEL
+
+struct hyperv_guid {
+uint8_t hv_guid[16];
+} __packed;
+
+#define HYPERV_GUID_STRLEN  40
+
+typedef uint64_t(*hyperv_tc64_t)(void);
+
+int hyperv_guid2str(const struct hyperv_guid *, char *,
+size_t);
+
+/*
+ * hyperv_tc64 could be NULL, if there were no suitable Hyper-V
+ * specific timecounter.
+ */
+extern hyperv_tc64_thyperv_tc64;
+extern u_inthyperv_features;/* CPUID_HV_MSR_ */
+
+#endif  /* _KERNEL */
+
+#endif  /* _HYPERV_H_ */
diff --git a/bsd/sys/dev/hyperv/include/hyperv_busdma.h  
b/bsd/sys/dev/hyperv/include/hyperv_busdma.h

--- a/bsd/sys/dev/hyperv/include/hyperv_busdma.h
+++ b/bsd/sys/dev/hyperv/include/hyperv_busdma.h
@@ -0,0 +1,49 @@
+/*-
+ * Copyright (c) 2016 Microsoft Corp.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *notice unmodified, this list of conditions, and the following
+ *disclaimer.
+ * 2. 

[COMMIT osv master] gcc 7: ignore warning in bsd/sys/netinet/in_mcast

[Commit Bot]

From: Nadav Har'El 
Committer: Nadav Har'El 
Branch: master

gcc 7: ignore warning in bsd/sys/netinet/in_mcast

In bsd/sys/netinet/in_mcast we get a "maybe uninitialized" warning which
as far as I can tell, is a false alarm. I also checked the freebsd tree,
and they haven't fixed anything in this area, so it's probably correct.

So let's just ignore this warning.

Signed-off-by: Nadav Har'El 
Message-Id: <20170712151334.12673-2-...@scylladb.com>

---
diff --git a/Makefile b/Makefile
--- a/Makefile
+++ b/Makefile
@@ -576,6 +576,7 @@ bsd += bsd/sys/netinet/in.o
 bsd += bsd/sys/netinet/in_pcb.o
 bsd += bsd/sys/netinet/in_proto.o
 bsd += bsd/sys/netinet/in_mcast.o
+$(out)/bsd/sys/netinet/in_mcast.o: COMMON += -Wno-maybe-uninitialized
 bsd += bsd/sys/netinet/in_rmx.o
 bsd += bsd/sys/netinet/ip_id.o
 bsd += bsd/sys/netinet/ip_icmp.o

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] gcc 7: add missing include

[Commit Bot]

From: Nadav Har'El 
Committer: Nadav Har'El 
Branch: master

gcc 7: add missing include

Gcc 7 complains about defining memset when we don't include a header
file with its prototype. So let's do.

Signed-off-by: Nadav Har'El 
Message-Id: <20170712151334.12673-1-...@scylladb.com>

---
diff --git a/fastlz/lzloader.cc b/fastlz/lzloader.cc
--- a/fastlz/lzloader.cc
+++ b/fastlz/lzloader.cc
@@ -5,6 +5,7 @@
  * BSD license as described in the LICENSE file in the top-level directory.
  */
 #include "fastlz.h"
+#include 
 #include 
 #include 
 #include 

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] gcc 7: fix use of uninitialized variable

[Commit Bot]

From: Nadav Har'El 
Committer: Nadav Har'El 
Branch: master

gcc 7: fix use of uninitialized variable

Gcc 7 rightly complains that in bsd_init() we create an uninitialized
timeval structure, and then use it to see the random number generator.
The intention was probably to fetch the current time here :-) So let's
do it.

Signed-off-by: Nadav Har'El 
Message-Id: <20170712151334.12673-3-...@scylladb.com>

---
diff --git a/bsd/init.cc b/bsd/init.cc
--- a/bsd/init.cc
+++ b/bsd/init.cc
@@ -44,6 +44,7 @@ void bsd_init(void)

 /* Random */
 struct timeval tv;
+gettimeofday(, NULL);
 bsd_srandom(tv.tv_sec ^ tv.tv_usec);

 arc4_init();

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] gcc 7: array should have non-negative length

[Commit Bot]

From: Nadav Har'El 
Committer: Nadav Har'El 
Branch: master

gcc 7: array should have non-negative length

Gcc 7 rightly complains when we try to allocate an array with a signed
int length. Changing the length variable to unsigned int fixes it.

Signed-off-by: Nadav Har'El 
Message-Id: <20170712151334.12673-4-...@scylladb.com>

---
diff --git a/core/app.cc b/core/app.cc
--- a/core/app.cc
+++ b/core/app.cc
@@ -297,7 +297,7 @@ void application::run_main(std::string path, int argc,  
char** argv)

 program_invocation_name = c_path;
 program_invocation_short_name = basename(c_path);

-auto sz = argc; // for the trailing 0's.
+unsigned sz = argc; // for the trailing 0's.
 for (int i = 0; i < argc; ++i) {
 sz += strlen(argv[i]);
 }

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] gcc 7: remove inappropriate use of "constexpr".

[Commit Bot]

From: Nadav Har'El 
Committer: Nadav Har'El 
Branch: master

gcc 7: remove inappropriate use of "constexpr".

In one test we assigned a constant-expression-but-not-marked-constexpr
pointer into a constexpr variable, which gcc 7 didn't like. There is no
need to bother with constexpr qualifications in tests so let's just
remove it.

Signed-off-by: Nadav Har'El 
Message-Id: <20170713072336.20818-1-...@scylladb.com>

---
diff --git a/tests/tst-mmap-file.cc b/tests/tst-mmap-file.cc
--- a/tests/tst-mmap-file.cc
+++ b/tests/tst-mmap-file.cc
@@ -107,8 +107,8 @@ int main(int argc, char *argv[])
 report(check_mapping(NULL, 0, MAP_SHARED, fd, 0, EINVAL) == 0,
 "force EINVAL by passing length equals to zero.");

-constexpr void *unaligned_addr = reinterpret_cast(0xefff1001);  
// unaligned 4k-sized page.
-constexpr void *aligned_addr   = reinterpret_cast(0xefff1000);  
// aligned 4k-sized page.
+void *unaligned_addr = reinterpret_cast(0xefff1001); //  
unaligned 4k-sized page.
+void *aligned_addr   = reinterpret_cast(0xefff1000); // aligned  
4k-sized page.
 // if MAP_FIXED was specified, then mmap should return EINVAL to  
unaligned addresses.
 report(check_mapping(unaligned_addr, size, MAP_SHARED | MAP_FIXED, fd,  
0, EINVAL) == 0,
 "force EINVAL by passing an unaligned addr and the flag  
MAP_FIXED.");


--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] gcc 7: fix "osv info threads"

[Commit Bot]

From: Nadav Har'El 
Committer: Nadav Har'El 
Branch: master

gcc 7: fix "osv info threads"

The internal structure of std::unique_ptr changed, and we need it to support
the "osv info threads" gdb command. With this patch both old and new layouts
will be supported.

Signed-off-by: Nadav Har'El 
Message-Id: <20170713072704.20976-1-...@scylladb.com>

---
diff --git a/scripts/loader.py b/scripts/loader.py
--- a/scripts/loader.py
+++ b/scripts/loader.py
@@ -942,7 +942,11 @@ def find_or_give_last(predicate, seq):
 return last

 def unique_ptr_get(u):
-return u['_M_t']['_M_head_impl']
+try:
+# Since gcc 7
+return u['_M_t']['_M_t']['_M_head_impl']
+except:
+return u['_M_t']['_M_head_impl']

 def thread_cpu(t):
 d = unique_ptr_get(t['_detached_state'])

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] Update apps submodule

[Commit Bot]

From: Nadav Har'El 
Committer: Nadav Har'El 
Branch: master

Update apps submodule

Signed-off-by: Nadav Har'El 

---
diff --git a/apps b/apps
--- a/apps
+++ b/apps
@@ -1 +1 @@
-Subproject commit b36e0d61f16a1b434ab02555443abc746ac92998
+Subproject commit 945ebfd812bf0d2d430f2e2ab606ae1bc1e7bfdc

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv-apps master] netperf: update download link

[Commit Bot]

From: Justin Cinkelj 
Committer: Nadav Har'El 
Branch: master

netperf: update download link

The netperf.org site was migrated to github.
The ftp.netperf.org is not even in DNS anymore.

Signed-off-by: Justin Cinkelj 
Message-Id: <20170628075435.22884-1-justin.cink...@xlab.si>

---
diff --git a/netperf/GET b/netperf/GET
--- a/netperf/GET
+++ b/netperf/GET
@@ -7,8 +7,9 @@ rm -rf upstream

 mkdir upstream
 cd upstream
-wget ftp://ftp.netperf.org/netperf/netperf-$VERSION.tar.bz2
-tar jxf netperf-$VERSION.tar.bz2
+wget  
https://codeload.github.com/HewlettPackard/netperf/zip/netperf-$VERSION -O  
netperf-$VERSION.zip

+unzip netperf-$VERSION.zip
+mv netperf-netperf-$VERSION netperf-$VERSION
 cd netperf-$VERSION

 ./configure

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] update apps/ submodule

[Commit Bot]

From: Nadav Har'El 
Committer: Nadav Har'El 
Branch: master

update apps/ submodule

Adds lighttpd application.

Signed-off-by: Nadav Har'El 

---
diff --git a/apps b/apps
--- a/apps
+++ b/apps
@@ -1 +1 @@
-Subproject commit aa725748ca15f9ca995105227245c53ab3ea4b7a
+Subproject commit b36e0d61f16a1b434ab02555443abc746ac92998

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] modules: add libcdio

[Commit Bot]

From: Justin Cinkelj 
Committer: Nadav Har'El 
Branch: master

modules: add libcdio

The iso-read from libcdio will be used to read ISO image
(contextualization CD, cloud metadata image). This will allow use of
standard ISO images in OSv cloud-init implementation, instead of using a
custom format (see 56c6195b252b127c7fe90cb9fa46c370b384b51a).

Signed-off-by: Justin Cinkelj 
Message-Id: <20170615103111.18174-2-justin.cink...@xlab.si>

---
diff --git a/modules/libcdio/.gitignore b/modules/libcdio/.gitignore
--- a/modules/libcdio/.gitignore
+++ b/modules/libcdio/.gitignore
@@ -0,0 +1,2 @@
+build
+download
diff --git a/modules/libcdio/Makefile b/modules/libcdio/Makefile
--- a/modules/libcdio/Makefile
+++ b/modules/libcdio/Makefile
@@ -0,0 +1,52 @@
+# A small Makefile "download and build" for libcdio
+
+# Directories for downloading and building
+DOWNLOAD=download
+BUILD=build
+
+# libcdio
+LIBCDIO_VERSION=0.94
+LIBCDIO_FOLDER=libcdio-$(LIBCDIO_VERSION)
+LIBCDIO_DOWNLOAD=https://ftp.gnu.org/gnu/libcdio/libcdio-$(LIBCDIO_VERSION).tar.gz
+LIBCDIO_ARCHIVE=download/$(LIBCDIO_FOLDER).tar.gz
+LIBCDIO_BUILD=$(BUILD)/$(LIBCDIO_FOLDER)
+
+CFLAGS:=-fPIC
+# libtool would ignore -shared anyway
+# We should set LDFLAGS=-shared. But then configure stage fails with
+# "configure: error: C compiler cannot create executables".
+# If we later edit Makefile to set LDFLAGS=-shared, then libtool removes  
it.

+# So, we will capture required cmd from make V=1, and relink the binary.
+LDFLAGS:=
+
+MAIN=$(LIBCDIO_BUILD)/src/.libs/iso-read
+
+module: $(MAIN) $(MAIN).so
+
+$(MAIN): $(LIBCDIO_BUILD)/Makefile
+   cd $(LIBCDIO_BUILD) && make
+
+$(MAIN).so: $(MAIN)
+   ./relink-binary.sh $@
+
+$(LIBCDIO_BUILD)/Makefile: $(LIBCDIO_BUILD)/configure
+   cd $(LIBCDIO_BUILD) && CFLAGS=$(CFLAGS) LDFLAGS=$(LDFLAGS) ./configure
+
+$(LIBCDIO_BUILD)/configure: $(LIBCDIO_ARCHIVE) | $(BUILD)
+   cd $(BUILD) && tar xzf ../$(LIBCDIO_ARCHIVE)
+   cd $(LIBCDIO_BUILD) && patch -p1 < ../../exit.patch
+   touch $(LIBCDIO_BUILD)/configure
+
+$(LIBCDIO_ARCHIVE): | $(DOWNLOAD)
+   cd $(DOWNLOAD) && \
+   curl --remote-name --remote-time $(LIBCDIO_DOWNLOAD)
+
+$(DOWNLOAD) $(BUILD):
+   @mkdir -p $@
+
+clean:
+   rm -rf $(BUILD) $(DOWNLOAD)
+
+
+
+.PHONY: module clean
diff --git a/modules/libcdio/exit.patch b/modules/libcdio/exit.patch
--- a/modules/libcdio/exit.patch
+++ b/modules/libcdio/exit.patch
@@ -0,0 +1,14 @@
+diff --git a/lib/driver/logging.c b/lib/driver/logging.c
+index b89ab37..202256c 100644
+--- a/lib/driver/logging.c
 b/lib/driver/logging.c
+@@ -49,7 +49,8 @@ cdio_default_log_handler(cdio_log_level_t level, const  
char message[])

+ fprintf (stderr, "**ERROR: %s\n", message);
+ fflush (stderr);
+   }
+-  exit (EXIT_FAILURE);
++  // Is triggered by invalid ISO image - input file smaller that  
about 32 kB

++  // exit (EXIT_FAILURE);
+   break;
+ case CDIO_LOG_DEBUG:
+   if (level >= cdio_loglevel_default) {
diff --git a/modules/libcdio/module.py b/modules/libcdio/module.py
--- a/modules/libcdio/module.py
+++ b/modules/libcdio/module.py
@@ -0,0 +1,13 @@
+from osv.modules.filemap import FileMap
+
+VERSION=0.94
+
+usr_files = FileMap()
+usr_files.add('${OSV_BASE}/modules/libcdio/build/libcdio-%s/src/.libs' %  
VERSION).to('/usr/bin') \

+   .include('iso-read.so')
+usr_files.add('${OSV_BASE}/modules/libcdio/build/libcdio-%s/lib/udf/.libs' %  
VERSION).to('/usr/lib')  
\

+   .include('libudf.so.0')
+usr_files.add('${OSV_BASE}/modules/libcdio/build/libcdio-%s/lib/iso9660/.libs' %  
VERSION).to('/usr/lib')  
\

+   .include('libiso9660.so.10')
+usr_files.add('${OSV_BASE}/modules/libcdio/build/libcdio-%s/lib/driver/.libs' %  
VERSION).to('/usr/lib')  
\

+   .include('libcdio.so.16')
diff --git a/modules/libcdio/relink-binary.sh  
b/modules/libcdio/relink-binary.sh

--- a/modules/libcdio/relink-binary.sh
+++ b/modules/libcdio/relink-binary.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -ev
+
+BIN=$1
+SRC_DIR=$(dirname "$BIN")/..
+SRC_FILE_BASE=$(basename "$BIN" | sed 's/\.so$//')
+SRC_FILE=$SRC_FILE_BASE.c
+
+cd $SRC_DIR
+touch $SRC_FILE
+CMD1=$(make V=1 | grep "\-o .libs/$SRC_FILE_BASE")
+CMD2=$(echo $CMD1 | sed -e "s|^libtool: link: ccache ||" -e "s|  
-o .libs/$SRC_FILE_BASE | -shared -o .libs/$SRC_FILE_BASE.so |")

+$CMD2

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] calloc: allow 0-sized allocation

[Commit Bot]

From: Justin Cinkelj 
Committer: Nadav Har'El 
Branch: master

calloc: allow 0-sized allocation

The calloc(nmemb, size=0) failed due to division by 0 in
"std::numeric_limits::max() / size".
Fix this by checking for 0 value before division.

Included is also a small test case for calloc.

Signed-off-by: Justin Cinkelj 
Message-Id: <20170615061205.16074-1-justin.cink...@xlab.si>

---
diff --git a/core/mempool.cc b/core/mempool.cc
--- a/core/mempool.cc
+++ b/core/mempool.cc
@@ -1564,6 +1564,8 @@ static inline void* std_malloc(size_t size, size_t  
alignment)


 void* calloc(size_t nmemb, size_t size)
 {
+if (nmemb == 0 || size == 0)
+return malloc(0);
 if (nmemb > std::numeric_limits::max() / size)
 return nullptr;
 auto n = nmemb * size;
diff --git a/modules/tests/Makefile b/modules/tests/Makefile
--- a/modules/tests/Makefile
+++ b/modules/tests/Makefile
@@ -86,7 +86,8 @@ tests := tst-pthread.so misc-ramdisk.so tst-vblk.so  
tst-bsd-evh.so \

tst-pthread-setcancelstate.so tst-syscall.so tst-pin.so tst-run.so \
tst-ifaddrs.so tst-pthread-affinity-inherit.so tst-sem-timed-wait.so \
tst-ttyname.so tst-pthread-barrier.so tst-feexcept.so tst-math.so \
-   tst-sigaltstack.so tst-fread.so tst-tcp-cork.so tst-tcp-v6.so
+   tst-sigaltstack.so tst-fread.so tst-tcp-cork.so tst-tcp-v6.so \
+   tst-calloc.so

 #  libstatic-thread-variable.so tst-static-thread-variable.so \

diff --git a/tests/tst-calloc.cc b/tests/tst-calloc.cc
--- a/tests/tst-calloc.cc
+++ b/tests/tst-calloc.cc
@@ -0,0 +1,67 @@
+/*
+* Copyright (C) 2017 XLAB d.o.o.
+*
+* This work is open source software, licensed under the terms of the
+* BSD license as described in the LICENSE file in the top-level directory.
+*/
+// To compile on Linux, use: g++ -g -pthread -std=c++11 tests/tst-calloc.cc
+
+#include 
+#include 
+
+// This test assures that the calloc function works as expected in the  
various

+// situations it can be used at.
+void pass_if(const char *got, const char *expected, int size, const char*  
msg)

+{
+if (memcmp(expected, got, size)) {
+std::cerr << "FAIL: " << msg << "\n";
+std::cerr << "ERROR: got ";
+std::cerr << got;
+std::cerr << " but expected ";
+std::cerr << expected;
+std::cerr << "\n";
+exit(1);
+}
+else {
+std::cerr << "PASS: " << msg << "\n";
+}
+}
+
+int main()
+{
+char *buf1;
+const char ref0[11] = "\0";
+const char refX[11] = "XX";
+int len1, len2;
+
+len1 = 1;
+len2 = 10;
+buf1 = (char*)calloc(len1, len2);
+pass_if(buf1, ref0, len1 * len2, "calloc(1, 10)");
+memset(buf1, 'X', len1 * len2);
+free(buf1);
+
+len1 = 2;
+len2 = 5;
+buf1 = (char*)calloc(len1, len2);
+pass_if(buf1, ref0, len1 * len2, "calloc(2, 5)");
+memset(buf1, 'X', len1 * len2);
+free(buf1);
+
+len1 = 1;
+len2 = 0;
+buf1 = (char*)calloc(len1, len2);
+pass_if(buf1, ref0, len1 * len2, "calloc(1, 0)");
+//memset(buf1, 'X', len1 * len2);
+free(buf1);
+
+len1 = 0;
+len2 = 1;
+buf1 = (char*)calloc(len1, len2);
+pass_if(buf1, ref0, len1 * len2, "calloc(0, 1)");
+memset(buf1, 'X', len1 * len2);
+free(buf1);
+
+std::cerr << "PASSED\n";
+return 0;
+}

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] fcntl: stub F_SETOWN

[Commit Bot]

From: Justin Cinkelj 
Committer: Nadav Har'El 
Branch: master

fcntl: stub F_SETOWN

fcntl(F_SETOWN) is used in nginx. We stub it so that nginx can be started
unmodified.

Signed-off-by: Justin Cinkelj 
Message-Id: <20170718091623.23284-3-justin.cink...@xlab.si>

---
diff --git a/fs/vfs/main.cc b/fs/vfs/main.cc
--- a/fs/vfs/main.cc
+++ b/fs/vfs/main.cc
@@ -1493,6 +1493,9 @@ int fcntl(int fd, int cmd, int arg)
 case F_SETLKW:
 WARN_ONCE("fcntl(F_SETLKW) stubbed\n");
 break;
+case F_SETOWN:
+WARN_ONCE("fcntl(F_SETOWN) stubbed\n");
+break;
 default:
 kprintf("unsupported fcntl cmd 0x%x\n", cmd);
 error = EINVAL;

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] signals: provide sigsuspend symbol

[Commit Bot]

From: Justin Cinkelj 
Committer: Nadav Har'El 
Branch: master

signals: provide sigsuspend symbol

sigsuspend is used in nginx. With current (not yet commited) patch, it
is never really called. Nginx master process code is used to execute
worker code, and this prevents master code to actually call sigsuspend.
So UNIMPL implementation is sufficient for now.

Signed-off-by: Justin Cinkelj 
Message-Id: <20170718114647.18016-1-justin.cink...@xlab.si>

---
diff --git a/libc/signal.cc b/libc/signal.cc
--- a/libc/signal.cc
+++ b/libc/signal.cc
@@ -213,6 +213,8 @@ int sigprocmask(int how, const sigset_t* _set,  
sigset_t* _oldset)

 return 0;
 }

+UNIMPL(int sigsuspend(const sigset_t *mask));
+
 int sigaction(int signum, const struct sigaction* act, struct sigaction*  
oldact)

 {
 // FIXME: We do not support any sa_flags besides SA_SIGINFO.

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv-apps master] Adjust grep expression used in lighttpd Makefile to filter out

[Commit Bot]

From: Waldemar Kozaczuk 
Committer: Nadav Har'El 
Branch: master

Adjust grep expression used in lighttpd Makefile to filter out

On some host systems (used to build) ldd tool returns list with invalid  
pointers

like so (the first line being the example):

linux-vdso.so.1 =>  (0x7ffc31fce000)
libpcre.so.3 => /lib/x86_64-linux-gnu/libpcre.so.3 (0x7fdea8035000)
libdl.so.2 => /lib/x86_64-linux-gnu/libdl.so.2 (0x7fdea7e31000)
libc.so.6 => /lib/x86_64-linux-gnu/libc.so.6 (0x7fdea7a66000)
	libpthread.so.0 => /lib/x86_64-linux-gnu/libpthread.so.0  
(0x7fdea7849000)

/lib64/ld-linux-x86-64.so.2 (0x55e5b2db)

This patch refines grep expression to remove lines like so.

Signed-off-by: Waldemar Kozaczuk 
Message-Id: <1500295438-30272-1-git-send-email-jwkozac...@gmail.com>

---
diff --git a/lighttpd/Makefile b/lighttpd/Makefile
--- a/lighttpd/Makefile
+++ b/lighttpd/Makefile
@@ -10,7 +10,7 @@ module: usr.manifest
 usr.manifest: $(SRC)/lighttpd.so
echo '[manifest]' > usr.manifest
echo '/lighttpd.so: $${MODULE_DIR}/$(SRC)/lighttpd.so' >> usr.manifest
-   ldd $(SRC)/lighttpd.so | grep -v 'libc.so\|libpthread.so\|libdl.so' |\
+	ldd $(SRC)/lighttpd.so | grep -v 'libc.so\|libpthread.so\|libdl.so\| 
=>\W*(' |\

sed -n 's/ *[^ ] *\(.*\) => \(.*\) .*/\/\1: \2/p' \
>> usr.manifest
 	ls upstream/lighttpd/src/.libs/*.so |  
sed 's/\(.*\)\/mod_\(.*\)/\/lighttpd\/lib\/mod_\2:  
$${MODULE_DIR}\/upstream\/lighttpd\/src\/.libs\/mod_\2/' >> usr.manifest


--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] lua: Add a check for openssl header version

[Commit Bot]

From: Benoît Canet 
Committer: Nadav Har'El 
Branch: master

lua: Add a check for openssl header version

If OpenSSL version is 1.1 prompt a message explaining
to the user how to install the compat headers.

Signed-off-by: Benoît Canet 
Message-Id: <20170725115124.14525-1-ben...@scylladb.com>

---
diff --git a/modules/lua/Makefile b/modules/lua/Makefile
--- a/modules/lua/Makefile
+++ b/modules/lua/Makefile
@@ -119,6 +119,8 @@ $(CDIR)/path.lua: $(LUA_ROCKS_BIN)
 LuaSec: $(CDIR)/ssl.lua

 $(CDIR)/ssl.lua: $(LUA_ROCKS_BIN)
+   ./check-openssl-version
+
 # Workaround because LuaRocks ignores /lib64
 ifneq ("$(wildcard /usr/lib64/libssl.so*)", "")
out/bin/luarocks install LuaSec 0.5 OPENSSL_LIBDIR=/usr/lib64
diff --git a/modules/lua/check-openssl-version  
b/modules/lua/check-openssl-version

--- a/modules/lua/check-openssl-version
+++ b/modules/lua/check-openssl-version
@@ -0,0 +1,16 @@
+#!/bin/sh
+
+# This is here to help cope with the transition period that the distro
+# use for openssl-1.0 to openssl-1.1
+if [ -f /usr/include/openssl/opensslv.h ]; then
+has_new_openssl=$(grep "OpenSSL 1.1" /usr/include/openssl/opensslv.h)
+fi
+if [ "$has_new_openssl"x != "x" ]; then
+echo "OpenSSL 1.1 header detected."
+echo "On Fedora >= 26 please use the compat headers by doing:"
+echo ""
+echo "dnf install --allowerasing compat-openssl10-devel -y"
+echo ""
+exit 1
+fi
+exit 0

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] libc: add tcflow

[Commit Bot]

From: Justin Cinkelj 
Committer: Nadav Har'El 
Branch: master

libc: add tcflow

At least a symbol needs to be defined for python2.7.

Signed-off-by: Justin Cinkelj 
Message-Id: <20170721104513.9854-2-justin.cink...@xlab.si>

---
diff --git a/Makefile b/Makefile
--- a/Makefile
+++ b/Makefile
@@ -1663,6 +1663,8 @@ libc += time/wcsftime.o
 libc += time/ftime.o # verbatim copy of the file as in 4b15d9f46a2b@musl
 $(out)/libc/time/ftime.o: CFLAGS += -Ilibc/include

+musl += termios/tcflow.o
+
 musl += unistd/sleep.o
 musl += unistd/gethostname.o
 libc += unistd/sethostname.o

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] Update apps submodule

[Commit Bot]

From: Nadav Har'El 
Committer: Nadav Har'El 
Branch: master

Update apps submodule

Adds the "python27" application

Signed-off-by: Nadav Har'El 

---
diff --git a/apps b/apps
--- a/apps
+++ b/apps
@@ -1 +1 @@
-Subproject commit ac345665ca8aa72a8e84d54d6f919d8e1d4f9b30
+Subproject commit fb9c20b1bd77f49aaecca63dcea983b6ae5198f0

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] libc: add __mbrlen

[Commit Bot]

From: Justin Cinkelj 
Committer: Nadav Har'El 
Branch: master

libc: add __mbrlen

At least a symbol needs to be defined for python2.7.
File mbrlen.c is moved from musl/ to libc/, as the weak alias declaration
must be in same file as function definition.

Signed-off-by: Justin Cinkelj 
Message-Id: <20170721104513.9854-3-justin.cink...@xlab.si>

---
diff --git a/Makefile b/Makefile
--- a/Makefile
+++ b/Makefile
@@ -1331,7 +1331,7 @@ libc += misc/__longjmp_chk.o
 musl += multibyte/btowc.o
 musl += multibyte/internal.o
 musl += multibyte/mblen.o
-musl += multibyte/mbrlen.o
+libc += multibyte/mbrlen.o
 musl += multibyte/mbrtowc.o
 musl += multibyte/mbsinit.o
 musl += multibyte/mbsnrtowcs.o
diff --git a/libc/multibyte/mbrlen.c b/libc/multibyte/mbrlen.c
--- a/libc/multibyte/mbrlen.c
+++ b/libc/multibyte/mbrlen.c
@@ -0,0 +1,20 @@
+/*
+ * This code was written by Rich Felker in 2010; no copyright is claimed.
+ * This code is in the public domain. Attribution is appreciated but
+ * unnecessary.
+ */
+
+#include 
+#include 
+#include 
+#include 
+
+//#include "internal.h"
+
+size_t mbrlen(const char *restrict s, size_t n, mbstate_t *restrict st)
+{
+   static unsigned internal;
+   return mbrtowc(0, s, n, st ? st : (mbstate_t *));
+}
+
+size_t __mbrlen (const char *, size_t, mbstate_t *) __attribute__ ((weak,  
alias ("mbrlen")));


--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] Use musl's unchanged mbsrtowcs()

[Commit Bot]

From: Nadav Har'El 
Committer: Nadav Har'El 
Branch: master

Use musl's unchanged mbsrtowcs()

Our change from musl's mbsrtowcs() is unnecessary, it's just taken from a
slightly older version of musl.

Signed-off-by: Nadav Har'El 

---
diff --git a/Makefile b/Makefile
--- a/Makefile
+++ b/Makefile
@@ -1335,7 +1335,7 @@ musl += multibyte/mbrlen.o
 musl += multibyte/mbrtowc.o
 musl += multibyte/mbsinit.o
 musl += multibyte/mbsnrtowcs.o
-libc += multibyte/mbsrtowcs.o
+musl += multibyte/mbsrtowcs.o
 musl += multibyte/mbstowcs.o
 musl += multibyte/mbtowc.o
 musl += multibyte/wcrtomb.o
diff --git a/libc/multibyte/mbsrtowcs.c b/libc/multibyte/mbsrtowcs.c
--- a/libc/multibyte/mbsrtowcs.c
+++ b/libc/multibyte/mbsrtowcs.c
@@ -1,100 +0,0 @@
-/*
- * This code was written by Rich Felker in 2010; no copyright is claimed.
- * This code is in the public domain. Attribution is appreciated but
- * unnecessary.
- */
-
-#include 
-#include 
-#include 
-#include 
-
-#include "internal.h"
-
-size_t mbsrtowcs(wchar_t *restrict ws, const char **restrict src, size_t  
wn, mbstate_t *restrict st)

-{
-   const unsigned char *s = (const void *)*src;
-   size_t wn0 = wn;
-   unsigned c = 0;
-
-   if (st && (c = *(unsigned *)st)) {
-   if (ws) {
-   *(unsigned *)st = 0;
-   goto resume;
-   } else {
-   goto resume0;
-   }
-   }
-
-   if (!ws) for (;;) {
-   if (*s-1u < 0x7f && (uintptr_t)s%4 == 0) {
-   while (!(( *(uint32_t*)s | *(uint32_t*)s-0x01010101) & 
0x80808080)) {
-   s += 4;
-   wn -= 4;
-   }
-   }
-   if (*s-1u < 0x7f) {
-   s++;
-   wn--;
-   continue;
-   }
-   if (*s-SA > SB-SA) break;
-   c = bittab[*s++-SA];
-resume0:
-   if (OOB(c,*s)) { s--; break; }
-   s++;
-   if (c&(1U<<25)) {
-   if (*s-0x80u >= 0x40) { s-=2; break; }
-   s++;
-   if (c&(1U<<19)) {
-   if (*s-0x80u >= 0x40) { s-=3; break; }
-   s++;
-   }
-   }
-   wn--;
-   c = 0;
-   } else for (;;) {
-   if (!wn) return wn0;
-   if (*s-1u < 0x7f && (uintptr_t)s%4 == 0) {
-			while (wn>=4 && !(( *(uint32_t*)s | *(uint32_t*)s-0x01010101) &  
0x80808080)) {

-   *ws++ = *s++;
-   *ws++ = *s++;
-   *ws++ = *s++;
-   *ws++ = *s++;
-   wn -= 4;
-   }
-   }
-   if (*s-1u < 0x7f) {
-   *ws++ = *s++;
-   wn--;
-   continue;
-   }
-   if (*s-SA > SB-SA) break;
-   c = bittab[*s++-SA];
-resume:
-   if (OOB(c,*s)) { s--; break; }
-   c = (c<<6) | *s++-0x80;
-   if (c&(1U<<31)) {
-   if (*s-0x80u >= 0x40) { s-=2; break; }
-   c = (c<<6) | *s++-0x80;
-   if (c&(1U<<31)) {
-   if (*s-0x80u >= 0x40) { s-=3; break; }
-   c = (c<<6) | *s++-0x80;
-   }
-   }
-   *ws++ = c;
-   wn--;
-   c = 0;
-   }
-
-   if (!c && !*s) {
-   if (ws) {
-   *ws = 0;
-   *src = 0;
-   }
-   return wn0-wn;
-   }
-   errno = EILSEQ;
-   if (ws) *src = (const void *)s;
-   return -1;
-}

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv-apps master] Add python2.7

[Commit Bot]

From: Justin Cinkelj 
Committer: Nadav Har'El 
Branch: master

Add python2.7

Based on @gberginc work.

Usage example:
./scripts/run.py -e "/python"
./scripts/run.py -e "/python -c \"aa={1:22,3:44}; print aa; print 'asdf'\""

Signed-off-by: Justin Cinkelj 
Message-Id: <20170722230157.10742-1-justin.cink...@xlab.si>

---
diff --git a/python27/Makefile b/python27/Makefile
--- a/python27/Makefile
+++ b/python27/Makefile
@@ -0,0 +1,52 @@
+VERSION=2.7
+SOURCE=https://github.com/python/cpython.git
+
+PYTHON_PREFIX=$(PWD)/src/python-install
+
+
+.PHONY: module clean ROOTFS
+
+SRC=src/cpython
+
+module: usr.manifest
+
+usr.manifest: ROOTFS
+   echo '/**: $${MODULE_DIR}/ROOTFS/**' > usr.manifest
+
+clean:
+   rm -fr usr.manifest ROOTFS src
+
+# Note: the touch commands below are needed after commands which create  
files
+# "in the past", like wget and tar, and can confuse Make to rebuild a  
target

+# which is new, just pretends to be old.
+$(SRC)/.git/HEAD:
+   mkdir src
+   cd src; git clone $(SOURCE)
+   cd $(SRC); git checkout -b $(VERSION) origin/$(VERSION)
+   #touch $@
+
+$(SRC)/Makefile: $(SRC)/.git/HEAD
+   cd $(SRC); ./configure --enable-shared --prefix=$(PYTHON_PREFIX)
+   cd $(SRC); cp Makefile Makefile.orig;
+   cd $(SRC); sed -i 's/^LDFLAGS=.*/LDFLAGS=-shared/g' Makefile
+
+$(SRC)/python: $(SRC)/Makefile
+   cd $(SRC); $(MAKE) python
+   cd $(SRC); mv python python.so
+   # Re-link python, this time as ELF executable.
+   # This is required for make install to succeed.
+   cd $(SRC); $(MAKE) -f Makefile.orig install
+
+ROOTFS: $(SRC)/python
+   rm ROOTFS -rf
+   mkdir -p ROOTFS/usr/lib
+
+   cp $(SRC)/python.so ROOTFS/python
+   cp $(PYTHON_PREFIX)/lib/libpython$(VERSION).so.1.0 ROOTFS/usr/lib
+	# Copy the Python environment. Since we'd like to exclude some dirs,  
rsync is used
+	rsync -a $(PYTHON_PREFIX)/lib/python$(VERSION) ROOTFS/lib --exclude test  
--exclude unittest \
+		--exclude '*.pyc' --exclude '*.pyo' --exclude '*.egg-info'  
--exclude '*.o' --exclude 'libpython'$(VERSION)'.a'
+	# TODO maybe .so files in ROOTFS/lib/python2.7/lib-dynload should be  
stripped.

+   ./copy-required-system-libraries.sh
+
+.DELETE_ON_ERROR:
diff --git a/python27/copy-required-system-libraries.sh  
b/python27/copy-required-system-libraries.sh

--- a/python27/copy-required-system-libraries.sh
+++ b/python27/copy-required-system-libraries.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+
+# Copy required libraries from the build system.
+# Likely, we don't want to include all libs (and their dependencies) from
+# ldd ROOTFS/lib/python2.7/lib-dynload/*so | grep -v -e '^ROOTFS/' |  
sed 's/(0x.*)//' | sort | uniq | awk '{print $3}'
+# So for now, only two (most obviously required) dependency libs will be  
included.

+
+set -e
+
+PYLIBS=""
+PYLIBS+=" ROOTFS/lib/python2.7/lib-dynload/readline.so "
+PYLIBS+=" ROOTFS/lib/python2.7/lib-dynload/_curses.so "
+
+DEPLIBS=""
+for pylib in $PYLIBS
+do
+	DEPLIBS+=`ldd $pylib | egrep -v -e linux-vdso.so -e ld-linux-x86-64.so  
-e 'lib(python[23]\.[0-9]|pthread|c|dl|m|util|).so.*'`

+   DEPLIBS+="\n"
+done
+#echo -e "DEPLIBS1:\n$DEPLIBS"
+DEPLIBS=`echo -e "$DEPLIBS" | sed '/^$/d' | awk '{print $3}' | sort | uniq`
+#echo -e "DEPLIBS2:\n$DEPLIBS"
+
+for deplib in $DEPLIBS
+do
+   /bin/cp $deplib ROOTFS/usr/lib
+done

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] Revert "lua: Upgrade LuaSec"

[Commit Bot]

From: Nadav Har'El 
Committer: Nadav Har'El 
Branch: master

Revert "lua: Upgrade LuaSec"

Compilation fails on both Fedora 25 and Fedora 26.
Benoit asked to try a different approach.

This reverts commit dc447e66dea85d283fa98425837f1c983adc9067.

---
diff --git a/modules/lua/Makefile b/modules/lua/Makefile
--- a/modules/lua/Makefile
+++ b/modules/lua/Makefile
@@ -121,13 +121,9 @@ LuaSec: $(CDIR)/ssl.lua
 $(CDIR)/ssl.lua: $(LUA_ROCKS_BIN)
 # Workaround because LuaRocks ignores /lib64
 ifneq ("$(wildcard /usr/lib64/libssl.so*)", "")
-   rm -rf luasec
-   git clone https://github.com/brunoos/luasec.git
-	cd luasec && ../out/bin/luarocks make luasec-0.6-1.rockspec  
OPENSSL_LIBDIR=/usr/lib64

+   out/bin/luarocks install LuaSec 0.5 OPENSSL_LIBDIR=/usr/lib64
 else
-   rm -rf luasec
-   git clone https://github.com/brunoos/luasec.git
-   cd luasec && ../out/bin/luarocks make luasec-0.6-1.rockspec
+   out/bin/luarocks install LuaSec 0.5
 endif

 ##

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] setup: Make Fedora26 take the compat openssl header

[Commit Bot]

From: Benoît Canet 
Committer: Nadav Har'El 
Branch: master

setup: Make Fedora26 take the compat openssl header

Using the default openssl-devel package would break
lua sec and start a dependency mess.
Use the compatibility header package so the build
and run will work just fine.

Signed-off-by: Benoît Canet 
Message-Id: <20170724130703.7055-2-ben...@scylladb.com>

---
diff --git a/scripts/setup.py b/scripts/setup.py
--- a/scripts/setup.py
+++ b/scripts/setup.py
@@ -29,63 +29,63 @@ class Fedora(object):
 'flex', 'bison', 'maven-shade-plugin', 'python-dpkt', 
'tcpdump', 'gdb',
 'gnutls-utils', 'openssl', 'p11-kit', 'patch', 'wget',
 'unzip', 'ncurses', 'ncurses-devel', 'libstdc++-static', 
'openssl-libs',
-'openssl-devel', 'libedit-devel', 'yaml-cpp-devel'
+'libedit-devel', 'yaml-cpp-devel'
 ]
 ec2_packages = standard_ec2_packages
 test_packages = ['openssl-devel']
 ec2_post_install = standard_ec2_post_install

 class Fedora_19(object):
-packages = ['java-1.7.0-openjdk', 'python-requests']
+packages =  
['java-1.7.0-openjdk', 'python-requests', 'openssl-devel']

 ec2_packages = []
 test_packages = []
 ec2_post_install = None
 version = '19'

 class Fedora_20(object):
-packages = ['java-1.7.0-openjdk', 'python-requests']
+packages =  
['java-1.7.0-openjdk', 'python-requests', 'openssl-devel']

 ec2_packages = []
 test_packages = []
 ec2_post_install = None
 version = '20'

 class Fedora_21(object):
-packages = ['java-1.7.0-openjdk', 'python-requests']
+packages =  
['java-1.7.0-openjdk', 'python-requests', 'openssl-devel']

 ec2_packages = []
 test_packages = []
 ec2_post_install = None
 version = '21'

 class Fedora_22(object):
-packages = ['java-1.8.0-openjdk', 'python-requests']
+packages =  
['java-1.8.0-openjdk', 'python-requests', 'openssl-devel']

 ec2_packages = []
 test_packages = []
 ec2_post_install = None
 version = '22'

 class Fedora_23(object):
-packages = ['java-1.8.0-openjdk', 'python2-requests']
+packages =  
['java-1.8.0-openjdk', 'python2-requests', 'openssl-devel']

 ec2_packages = []
 test_packages = []
 ec2_post_install = None
 version = '23'

 class Fedora_24(object):
-packages = ['java-1.8.0-openjdk', 'python2-requests']
+packages =  
['java-1.8.0-openjdk', 'python2-requests', 'openssl-devel']

 ec2_packages = []
 test_packages = []
 ec2_post_install = None
 version = '24'

 class Fedora_25(object):
-packages = ['java-1.8.0-openjdk', 'python2-requests']
+packages =  
['java-1.8.0-openjdk', 'python2-requests', 'openssl-devel']

 ec2_packages = []
 test_packages = []
 ec2_post_install = None
 version = '25'

 class Fedora_26(object):
-packages = ['java-1.8.0-openjdk', 'python2-requests']
+packages =  
['java-1.8.0-openjdk', 'python2-requests', 'compat-openssl10-devel']

 ec2_packages = []
 test_packages = []
 ec2_post_install = None

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] gcc 7: fix crashes due to corruption of xmm0 register

[Commit Bot]

From: Nadav Har'El 
Committer: Nadav Har'El 
Branch: master

gcc 7: fix crashes due to corruption of xmm0 register

This patch fixes #890, which reported multiple cases of early crashes of OSv
when compiled with gcc 7. The common feature of all these crashes was that
a small random part of memory got zeroed, with various sad consequences.

The cause of these crashes is failing to correctly save FPU state. In modern
code the FPU (and SSE et al.) is used not just for floating point  
operations,

but also for various vector operations generated by the compiler as an
alternative to multiple instructions. Corrupting the FPU state can cause
such vector operations to go haywire, including zeroing the wrong place or
length of memory memory.

In some places, we save the FPU state on the current stack, using the
"fpu_lock" class. That class's constructor calls fpu_state_init() to
initialize the save area, and then fpu_state_save()/fpu_state_restore()
to save and restore the state.

Because fpu_state_init() is called before saving the FPU state, it must not
modify any of the FPU registers. But unfortunately, in gcc 7, it did: With
O2 optimization enabled, it zeroed the xmm0 register:

The XSAVE-based variant of fpu_state_init() (fpu_state_init_xsave()) wants
to zero 24 bytes of memory. It calls memset(), but the compiler optimizes
this call away and replaces it with direct instructions to zero 24 bytes.
Prior to gcc 7, this generated 3 MOVQ instructions, totalling 33 bytes of
machine code:

 48 c7 87 00 02 00 00movq   $0x0,0x200(%rdi)
 00 00 00 00
 48 c7 87 08 02 00 00movq   $0x0,0x208(%rdi)
 00 00 00 00
 48 c7 87 10 02 00 00movq   $0x0,0x210(%rdi)
 00 00 00 00

starting with gcc 7, a significantly more elaborate technique using SSE
registers is used to zero the first 16 bytes. The result is only 22 bytes
of code, saving a "whopping" 11 bytes of code:

 66 0f ef c0 pxor   %xmm0,%xmm0
 48 c7 87 10 02 00 00movq   $0x0,0x210(%rdi)
 00 00 00 00
 0f 11 87 00 02 00 00movups %xmm0,0x200(%rdi)

In the case of fpu_state_init_xsave(), we of course do not care about
saving of 11 bytes of code, but the function is ruined by using SSE
registers. So this patch adds a tag on this function telling gcc to not
use SSE when compiling this function.

P.S. This patch has to be a new world record. Zero lines of code,
and 50 lines of explanations :-)

Signed-off-by: Nadav Har'El 
Message-Id: <20170723195843.23951-1-...@scylladb.com>

---
diff --git a/arch/x64/arch-cpu.cc b/arch/x64/arch-cpu.cc
--- a/arch/x64/arch-cpu.cc
+++ b/arch/x64/arch-cpu.cc
@@ -43,7 +43,14 @@ exception_guard::~exception_guard()
 }

 extern "C"
+[[gnu::target("no-sse")]]
 void fpu_state_init_xsave(processor::fpu_state *s) {
+// s->xsavehdr is known at compile time, so gcc will not call memset()
+// here. Rather it will generate direct instructions to zero the given
+// length (24 bytes). We can't allow the compiler to use any SSE  
registers
+// for that, because this function is used in fpu_lock before saving  
the

+// FPU state to the stack, so must not touch any of the FPU registers.
+// This is why the "gnu::target(no-sse)" specification above is  
critical.

 memset(s->xsavehdr, 0, sizeof(s->xsavehdr));
 }

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] run.py: drop unnecessary qemu option

[Commit Bot]

From: Nadav Har'El 
Committer: Nadav Har'El 
Branch: master

run.py: drop unnecessary qemu option

Recent versions of qemu complain about the do-nothing and deprecated
"default" option to "-mon". We can just get rid of it.

Signed-off-by: Nadav Har'El 

---
diff --git a/scripts/run.py b/scripts/run.py
--- a/scripts/run.py
+++ b/scripts/run.py
@@ -183,7 +183,7 @@ def start_osv_qemu(options):
 else:
 signal_option = ('off', 'on')[options.with_signals]
 args += ["-chardev", "stdio,mux=on,id=stdio,signal=%s" %  
signal_option]

-args += ["-mon", "chardev=stdio,mode=readline,default"]
+args += ["-mon", "chardev=stdio,mode=readline"]
 args += ["-device", "isa-serial,chardev=stdio"]

 for a in options.pass_args or []:

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] Fix compilation error on older compilers

[Commit Bot]

From: HawxChen 
Committer: Avi Kivity 
Branch: master

Fix compilation error on older compilers

Complete the commit: a675ae787660648a22cef984522e27bafb398bc8.

Fixes #896.

Signed-off-by: HawxChen 
Message-Id: <1501102889-14514-1-git-send-email-yingjheng.c...@gmail.com>
Signed-off-by: Avi Kivity 

---
diff --git a/include/osv/aligned_new.hh b/include/osv/aligned_new.hh
--- a/include/osv/aligned_new.hh
+++ b/include/osv/aligned_new.hh
@@ -57,7 +57,7 @@ T* aligned_array_new(size_t len) {

 template
 void aligned_array_delete(T* p) {
-static_assert(sizeof(T) > sizeof(size_t));
+static_assert(sizeof(T) > sizeof(size_t), "small T in  
aligned_array_new");

 size_t len = *(size_t*)p;
 for (unsigned i = 0; i < len ; i++) {
 ++p;

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] Fix compilation error on older compilers

[Commit Bot]

From: Nadav Har'El 
Committer: Nadav Har'El 
Branch: master

Fix compilation error on older compilers

Allowing a static_assert without a separate message string is,
unfortunately, a C++17 extension. Older compiler needs this string,
as silly as it is.

Fixes #896.

Signed-off-by: Nadav Har'El 

---
diff --git a/include/osv/aligned_new.hh b/include/osv/aligned_new.hh
--- a/include/osv/aligned_new.hh
+++ b/include/osv/aligned_new.hh
@@ -43,7 +43,7 @@ T* aligned_new(Args&&... args) {
 template
 T* aligned_array_new(size_t len) {
 // Allocate one extra item in the beginning, for length.
-static_assert(sizeof(T) > sizeof(size_t));
+static_assert(sizeof(T) > sizeof(size_t), "small T in  
aligned_array_new");

 void *p = aligned_alloc(alignof(T), sizeof(T) * (len+1));
 assert(p);
 *(size_t *)p = len;

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] update apps submodule

[Commit Bot]

From: Nadav Har'El 
Committer: Nadav Har'El 
Branch: master

update apps submodule

Signed-off-by: Nadav Har'El 

---
diff --git a/apps b/apps
--- a/apps
+++ b/apps
@@ -1 +1 @@
-Subproject commit 945ebfd812bf0d2d430f2e2ab606ae1bc1e7bfdc
+Subproject commit ac345665ca8aa72a8e84d54d6f919d8e1d4f9b30

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] pci-generic: free the device pointer when parse device config failure

[Commit Bot]

From: Dongjiu Geng 
Committer: Nadav Har'El 
Branch: master

pci-generic: free the device pointer when parse device config failure

when parse device config failure, it will break out of the circulation
loop, but not free the allocated device pointer. so this patch free that
pointer to avoid memory leakage

Signed-off-by: Dongjiu Geng 
Message-Id: <1500877699-12208-1-git-send-email-gengdong...@huawei.com>

---
diff --git a/drivers/pci-generic.cc b/drivers/pci-generic.cc
--- a/drivers/pci-generic.cc
+++ b/drivers/pci-generic.cc
@@ -88,6 +88,8 @@ bool check_bus(u16 bus)
 if (!parse_ok) {
 pci_e("Error: couldn't parse device config  
space %02x:%02x.%x",

 bus, slot, func);
+// free the dev pointer to avoid memory leakage
+delete dev;
 break;
 }

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] lua: Upgrade LuaSec

[Commit Bot]

From: Benoît Canet 
Committer: Nadav Har'El 
Branch: master

lua: Upgrade LuaSec

The LuaRock repository is not up to date - the latest release it has of
LuaSec is a year old and does not successfully compile on new distributions
with Gcc 7 and new openssl.

Use the github repository.

Signed-of-by: Benoît Canet 
Message-Id: <20170722105150.26589-1-ben...@scylladb.com>

---
diff --git a/modules/lua/Makefile b/modules/lua/Makefile
--- a/modules/lua/Makefile
+++ b/modules/lua/Makefile
@@ -121,9 +121,13 @@ LuaSec: $(CDIR)/ssl.lua
 $(CDIR)/ssl.lua: $(LUA_ROCKS_BIN)
 # Workaround because LuaRocks ignores /lib64
 ifneq ("$(wildcard /usr/lib64/libssl.so*)", "")
-   out/bin/luarocks install LuaSec 0.5 OPENSSL_LIBDIR=/usr/lib64
+   rm -rf luasec
+   git clone https://github.com/brunoos/luasec.git
+	cd luasec && ../out/bin/luarocks make luasec-0.6-1.rockspec  
OPENSSL_LIBDIR=/usr/lib64

 else
-   out/bin/luarocks install LuaSec 0.5
+   rm -rf luasec
+   git clone https://github.com/brunoos/luasec.git
+   cd luasec && ../out/bin/luarocks make luasec-0.6-1.rockspec
 endif

 ##

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] gcc 7: replacement for "new" for types of special alignment

[Commit Bot]

From: Nadav Har'El 
Committer: Nadav Har'El 
Branch: master

gcc 7: replacement for "new" for types of special alignment

OSv has various types which have special alignment requirements - e.g.,
some objects are aligned to cache-line size for performance, some objects
are aligned because of processor requirements (e.g., fpu state needs to
be 64-byte aligned) or ABI requirements (stacks need to be 16-byte aligned).

While C++11 correctly aligns these object on the stack, when created on
the heap with "new", it actually ignores the requested alignment!
The reason is that "new" calls "operator new()" which cannot specify the
desired alignment. We're lucky that in practice missing the alignment
didn't matter: In some cases the alignment was just supposed to be a
performance optimization that is not mandatory. In other cases like
the thread object, because it is large, it was already allocated using
entire pages and ended up getting a 4096 byte alignment, more than enough.

But Gcc starting in release 7, warns about using "new" on a type with a
non-standard alignment requirement. What we do in this patch is to create
a template function aligned_new(...) which can be used instead of new,
and uses aligned_alloc() to implement the properly aligned allocation.

When we switch to C++17 in the future, its "new" fully supports any  
alignment

(and to do that, it has a version of "operator new" taking an alignment -
see http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2012/n3396.htm)
so we will be able to revert this patch, and go back to using regular new.

Signed-off-by: Nadav Har'El 

---
diff --git a/arch/x64/smp.cc b/arch/x64/smp.cc
--- a/arch/x64/smp.cc
+++ b/arch/x64/smp.cc
@@ -21,6 +21,7 @@ extern "C" {
 #include 
 #include 
 #include "osv/percpu.hh"
+#include 

 extern "C" { void smp_main(void); }

@@ -76,7 +77,7 @@ void smp_init()
 parse_madt();
 sched::current_cpu = sched::cpus[0];
 for (auto c : sched::cpus) {
-c->incoming_wakeups = new  
sched::cpu::incoming_wakeup_queue[sched::cpus.size()];
+c->incoming_wakeups =  
aligned_array_new(sched::cpus.size());

 }
 smpboot_cr0 = read_cr0();
 smpboot_cr4 = read_cr4();
diff --git a/bsd/sys/netinet/tcp_input.cc b/bsd/sys/netinet/tcp_input.cc
--- a/bsd/sys/netinet/tcp_input.cc
+++ b/bsd/sys/netinet/tcp_input.cc
@@ -90,6 +90,7 @@
 #include 
 #include 
 #include 
+#include 

 TRACEPOINT(trace_tcp_input_ack, "%p: We've got ACK: %u", void*, unsigned  
int);


@@ -3226,7 +3227,7 @@ static ipv4_tcp_conn_id tcp_connection_id(tcpcb* tp)
 void
 tcp_setup_net_channel(tcpcb* tp, struct ifnet* intf)
 {
-	auto nc = new net_channel([=] (mbuf *m) { tcp_net_channel_packet(tp, m);  
});
+	auto nc = aligned_new([=] (mbuf *m) {  
tcp_net_channel_packet(tp, m); });

tp->nc = nc;
tp->nc_intf = intf;
intf->add_net_channel(nc, tcp_connection_id(tp));
diff --git a/core/async.cc b/core/async.cc
--- a/core/async.cc
+++ b/core/async.cc
@@ -18,6 +18,7 @@
 #include 
 #include 
 #include 
+#include 

 namespace async {

@@ -247,7 +248,7 @@ class async_worker {
 static PERCPU(async_worker*, _percpu_worker);

 static sched::cpu::notifier _notifier([] () {
-*_percpu_worker = new async_worker(sched::cpu::current());
+*_percpu_worker = aligned_new(sched::cpu::current());
 });

 static inline async_worker& get_worker()
diff --git a/drivers/virtio-net.cc b/drivers/virtio-net.cc
--- a/drivers/virtio-net.cc
+++ b/drivers/virtio-net.cc
@@ -861,7 +861,7 @@ hw_driver* net::probe(hw_device* dev)
 if (opt_maxnic && maxnic-- <= 0) {
 return nullptr;
 } else {
-return new net(*pci_dev);
+return aligned_new(*pci_dev);
 }
 }
 }
diff --git a/drivers/vmxnet3.cc b/drivers/vmxnet3.cc
--- a/drivers/vmxnet3.cc
+++ b/drivers/vmxnet3.cc
@@ -39,6 +39,7 @@

 #include 
 #include 
+#include 

 #include "drivers/clock.hh"
 #include "drivers/clockevent.hh"
@@ -428,7 +429,7 @@ hw_driver* vmxnet3::probe(hw_device* dev)
 if (opt_maxnic && maxnic-- <= 0) {
 return nullptr;
 } else {
-return new vmxnet3(*pci_dev);
+return aligned_new(*pci_dev);
 }
 }
 }
diff --git a/include/osv/aligned_new.hh b/include/osv/aligned_new.hh
--- a/include/osv/aligned_new.hh
+++ b/include/osv/aligned_new.hh
@@ -0,0 +1,68 @@
+/*
+ * Copyright (C) 2017 ScyllaDB
+ *
+ * This work is open source software, licensed under the terms of the
+ * BSD license as described in the LICENSE file in the top-level directory.
+ */
+
+#ifndef ALIGNED_NEW_HH
+#define ALIGNED_NEW_HH
+
+/**
+ * In C++11, "new T()" is not guaranteed to fulfill unusual alignment
+ * requirements that T might have. The problem is that there is no way to
+ * tell "operator new" (which "new T()" calls to allocate memory) to use
+ * a particular 

[COMMIT osv master] ramfs: support symbolic links

[Commit Bot]

From: Nadav Har'El 
Committer: Nadav Har'El 
Branch: master

ramfs: support symbolic links

This patch adds the missing support in ramfs for creating and reading
symbolic links, and then modifies the "bootfs" format (which we use to
pack and unpack the ram disk files) to also allow symbolic links.

After this patch, the following works:
scripts/build image=cli fs=ramfs

Fixes #871

Signed-off-by: Nadav Har'El 
Message-Id: <20170425225010.24418-2-...@scylladb.com>

---
diff --git a/fs/ramfs/ramfs_vnops.cc b/fs/ramfs/ramfs_vnops.cc
--- a/fs/ramfs/ramfs_vnops.cc
+++ b/fs/ramfs/ramfs_vnops.cc
@@ -224,6 +224,42 @@ ramfs_mkdir(struct vnode *dvp, char *name, mode_t mode)
return 0;
 }

+static int
+ramfs_symlink(struct vnode *dvp, char *name, char *link)
+{
+   auto np = ramfs_add_node((ramfs_node*)dvp->v_data, name, VLNK);
+   if (np == NULL)
+   return ENOMEM;
+   // Save the link target without the final null, as readlink() wants it.
+   size_t len = strlen(link);
+   np->rn_buf = strndup(link, len);
+   np->rn_bufsize = np->rn_size = len;
+   return 0;
+}
+
+static int
+ramfs_readlink(struct vnode *vp, struct uio *uio) {
+   struct ramfs_node *np = (ramfs_node*)vp->v_data;
+   size_t len;
+
+   if (vp->v_type != VLNK) {
+   return EINVAL;
+   }
+   if (uio->uio_offset < 0) {
+   return EINVAL;
+   }
+   if (uio->uio_resid == 0) {
+   return 0;
+   }
+   if (uio->uio_offset >= (off_t)vp->v_size)
+   return 0;
+   if (vp->v_size - uio->uio_offset < uio->uio_resid)
+   len = vp->v_size - uio->uio_offset;
+   else
+   len = uio->uio_resid;
+   return uiomove(np->rn_buf + uio->uio_offset, len, uio);
+}
+
 /* Remove a directory */
 static int
 ramfs_rmdir(struct vnode *dvp, struct vnode *vp, char *name)
@@ -388,11 +424,11 @@ ramfs_rename(struct vnode *dvp1, struct vnode *vp1,  
char *name1,

} else {
/* Create new file or directory */
old_np = (ramfs_node*)vp1->v_data;
-   np = ramfs_add_node((ramfs_node*)dvp2->v_data, name2, VREG);
+   np = ramfs_add_node((ramfs_node*)dvp2->v_data, name2, 
old_np->rn_type);
if (np == NULL)
return ENOMEM;

-   if (vp1->v_type == VREG) {
+   if (old_np->rn_buf) {
/* Copy file data */
np->rn_buf = old_np->rn_buf;
np->rn_size = old_np->rn_size;
@@ -439,6 +475,8 @@ ramfs_readdir(struct vnode *vp, struct file *fp, struct  
dirent *dir)

}
if (np->rn_type == VDIR)
dir->d_type = DT_DIR;
+   else if (np->rn_type == VLNK)
+   dir->d_type = DT_LNK;
else
dir->d_type = DT_REG;
strlcpy((char *)>d_name, np->rn_name,
@@ -476,8 +514,6 @@ ramfs_getattr(struct vnode *vnode, struct vattr *attr)
 #define ramfs_inactive ((vnop_inactive_t)vop_nullop)
 #define ramfs_link ((vnop_link_t)vop_eperm)
 #define ramfs_fallocate ((vnop_fallocate_t)vop_nullop)
-#define ramfs_readlink ((vnop_readlink_t)vop_nullop)
-#define ramfs_symlink  ((vnop_symlink_t)vop_nullop)

 /*
  * vnode operations
diff --git a/fs/vfs/main.cc b/fs/vfs/main.cc
--- a/fs/vfs/main.cc
+++ b/fs/vfs/main.cc
@@ -2100,11 +2100,21 @@ int chroot(const char *path)
 return -1;
 }

-#define BOOTFS_PATH_MAX 112
-
+// unpack_bootfs() unpacks a collection of files stored as part of the OSv
+// executable (in memory location "bootfs_start") into the file system,
+// normally the in-memory filesystem ramfs.
+// The files are packed in the executable in an ad-hoc format defined here.
+// Code in scripts/mkbootfs.py packs files into this format.
+#define BOOTFS_PATH_MAX 111
+enum class bootfs_file_type : char { other = 0, symlink = 1 };
 struct bootfs_metadata {
 uint64_t size;
 uint64_t offset;
+// The file's type. Can be "symlink" or "other". A directory is  
an "other"

+// file with its name ending with a "/" (and no content).
+bootfs_file_type type;
+// name must end with a null. For symlink files, the content must end
+// with a null as well.
 char name[BOOTFS_PATH_MAX];
 };

@@ -2130,6 +2140,15 @@ void unpack_bootfs(void)
 }
 }

+if (md[i].type == bootfs_file_type::symlink) {
+// This is a symbolic link record. The file's content is the
+// target path, and we assume ends with a null.
+if (symlink(_start + md[i].offset, md[i].name) != 0) {
+kprintf("couldn't symlink %s: %d\n", md[i].name, errno);
+sys_panic("unpack_bootfs failed");
+}
+continue;
+}
 if (*(p-1) == '/' && md[i].size == 0) {
 // This is 

[COMMIT osv-apps master] Add python2.7 from the build system

[Commit Bot]

From: Justin Cinkelj 
Committer: Nadav Har'El 
Branch: master

Add python2.7 from the build system

Instead of compiling python from source, we reuse libpython2.7.so
from the build system. Only the python executable is recompiled as
shared object. The python.c file is almost literal copy from the git
source (only 'include "Python.h"' is replaced with declaration of
Py_Main function). The gcc command to compile python.so includes
-lpython2.7. This likely requires python-devel to be installed on
build system.

The GET file includes all required shared libraries. The install_shlibs()
is called 4 times, until no new libs are required (actually, it only
checks that no new libs were added - should be good enough).

usr.manifest adds symlink /lib64 to /lib. This is required to run
python simply as:
./scripts/run.py -e "/python"
./scripts/run.py -e "/python -c \"aa={1:22,3:44}; print aa; print 'asdf'\""
Otherwise, it was required to set PYTHONPATH, (like in
./scripts/run.py -e "--env=PYTHONPATH=/lib/python2.7/ /python").

Signed-off-by: Justin Cinkelj 
Message-Id: <20170801114115.2452-3-justin.cink...@xlab.si>

---
diff --git a/python27/GET b/python27/GET
--- a/python27/GET
+++ b/python27/GET
@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+set -e
+
+VERSION=2.7
+BASEDIR=$PWD
+ROOTFS=$BASEDIR/ROOTFS
+
+install_shlibs() {
+SHLIBS=""
+SHLIBS+=" $ROOTFS/python.so "
+SHLIBS+=" `find $ROOTFS -iname '*\.so' | grep -v '/site-packages/'` "
+set +e
+SHLIBS+=" `find $ROOTFS -iname '*\.so[\.0-9]*' | grep  
-v '/site-packages/'` "

+set -e
+SHLIBS_COUNT="`echo \"$SHLIBS\" | wc -l`"
+
+ldd $SHLIBS | grep -Po '(?<=> )/[^ ]+' | sort | uniq | grep -Pv 'lib(c|gcc| 
dl|m|util|rt|pthread|stdc\+\+|selinux|krb5|gssapi_krb5)\.so' | xargs -I {}  
install  {} $ROOTFS/usr/lib
+# ROOTFS/lib/python2.7/config/libpython2.7.so is a symlink  
to ../../libpython2.7.so,

+# so create a valid destination to avoid ldd error due to dangling symlink.
+(cd $ROOTFS/lib && ln -sf ../usr/lib/libpython$VERSION.so.1.0  
libpython$VERSION.so)

+echo "$SHLIBS_COUNT"
+}
+
+main() {
+mkdir -p build/
+gcc -o build/python.so python.c -fPIC -shared -lpython${VERSION}
+
+rm -rf "$ROOTFS"
+mkdir -p "$ROOTFS/usr/lib"
+mkdir -p "$ROOTFS/lib/python$VERSION"
+
+cp build/python.so "$ROOTFS"
+install_shlibs
+# TODO /lib64/python2.7/ should not be hardcoded?
+PY_LIB1=/lib64/python$VERSION/
+rsync -a $PY_LIB1 $ROOTFS/lib/python$VERSION/ --exclude test --exclude  
unittest \

+--exclude '*.pyc' --exclude '*.pyo' --exclude '*.egg-info'
+
+SHLIBS_COUNT4=`install_shlibs`
+echo "Python SHLIBS_COUNT4=$SHLIBS_COUNT4"
+SHLIBS_COUNT3=`install_shlibs`
+echo "Python SHLIBS_COUNT3=$SHLIBS_COUNT3"
+SHLIBS_COUNT2=`install_shlibs`
+echo "Python SHLIBS_COUNT2=$SHLIBS_COUNT2"
+SHLIBS_COUNT1=`install_shlibs`
+echo "Python SHLIBS_COUNT1=$SHLIBS_COUNT1"
+if [ $SHLIBS_COUNT1 -ne $SHLIBS_COUNT2 ]
+then
+   # if this happens, just add additional calls to install_shlibs()
+   echo "ERROR some libraries required by python might be missing"
+   exit 1
+fi
+}
+
+main
diff --git a/python27/Makefile b/python27/Makefile
--- a/python27/Makefile
+++ b/python27/Makefile
@@ -0,0 +1,8 @@
+
+.PHONY: module clean
+
+module:
+   ./GET
+
+clean:
+   rm -fr ROOTFS build
diff --git a/python27/python.c b/python27/python.c
--- a/python27/python.c
+++ b/python27/python.c
@@ -0,0 +1,22 @@
+/* Minimal main program -- everything is loaded from the library */
+
+//#include "Python.h"
+extern int Py_Main(int argc, char **argv);
+
+#ifdef __FreeBSD__
+#include 
+#endif
+
+int
+main(int argc, char **argv)
+{
+   /* 754 requires that FP exceptions run in "no stop" mode by default,
+* and until C vendors implement C99's ways to control FP exceptions,
+* Python requires non-stop mode.  Alas, some platforms enable FP
+* exceptions by default.  Here we disable them.
+*/
+#ifdef __FreeBSD__
+   fedisableexcept(FE_OVERFLOW);
+#endif
+   return Py_Main(argc, argv);
+}
diff --git a/python27/usr.manifest b/python27/usr.manifest
--- a/python27/usr.manifest
+++ b/python27/usr.manifest
@@ -0,0 +1,5 @@
+/python: ->/python.so
+# to remove need for --env=PYTHONPATH=/lib/python2.7/, as _my_ build  
system has python .py files in /lib64/python2.7/

+/lib64: ->/lib
+
+/**: ${MODULE_DIR}/ROOTFS/**

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] Update apps submodule

[Commit Bot]

From: Nadav Har'El 
Committer: Nadav Har'El 
Branch: master

Update apps submodule

Renames from-source python package to "python27-fromsource" and
a new "python27" package which takes the unmodified Python 2.7 from
the build machine.

Signed-off-by: Nadav Har'El 

---
diff --git a/apps b/apps
--- a/apps
+++ b/apps
@@ -1 +1 @@
-Subproject commit fb9c20b1bd77f49aaecca63dcea983b6ae5198f0
+Subproject commit 82381d13fb2ae832800596ba05a41d67135ce562

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv-apps master] apps: rename python27 to python27-fromsource

[Commit Bot]

From: Nadav Har'El 
Committer: Nadav Har'El 
Branch: master

apps: rename python27 to python27-fromsource

This version gets the Python 2.7 sources and compiles them. We will
want to also add a package which takes the one installed on this machine
instead of compiling it.

Signed-off-by: Nadav Har'El 

---
diff --git a/python27-fromsource/Makefile b/python27-fromsource/Makefile
--- a/python27-fromsource/Makefile
+++ b/python27-fromsource/Makefile
null
diff --git a/python27-fromsource/copy-required-system-libraries.sh  
b/python27-fromsource/copy-required-system-libraries.sh

--- a/python27-fromsource/copy-required-system-libraries.sh
+++ b/python27-fromsource/copy-required-system-libraries.sh
null

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv-apps master] python2.7: add README

[Commit Bot]

From: Nadav Har'El 
Committer: Nadav Har'El 
Branch: master

python2.7: add README

Signed-off-by: Nadav Har'El 

---
diff --git a/python27/README b/python27/README
--- a/python27/README
+++ b/python27/README
@@ -0,0 +1,8 @@
+This app builds an image containing the already-compiled Python shared
+object (libpython2.7.so) installed on the build macine, and some of
+the required libraries from the build machine.
+
+Example usage:
+
+./scripts/run.py -e "/python"
+./scripts/run.py -e "/python -c \"aa={1:22,3:44}; print aa; print 'asdf'\""

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] ELF: fix assertion failure when trying to run short non-executable

[Commit Bot]

From: Nadav Har'El 
Committer: Nadav Har'El 
Branch: master

ELF: fix assertion failure when trying to run short non-executable

When trying to run a short non-executable (e.g. "scripts/run.py -e  
/etc/hosts")

we used to fail with an assertion failure in the function read(fileref, ...)
from fs/fs.cc. This function assumes it *can* read the desired buffer, and
if it can't (because the file is too short), it crashes. So when we try to
read the executable's header in a file which is shorter than the expected
size of such header, we crashed.

The solution is simple - verify that the size of the file is big enough
before calling the read function. If it is not, throw an exception.

This exception (and all the other exceptions in core/elf.cc during loading
an object) should be converted to a new type, as explained in issue #904.

Signed-off-by: Nadav Har'El 

---
diff --git a/core/elf.cc b/core/elf.cc
--- a/core/elf.cc
+++ b/core/elf.cc
@@ -267,6 +267,11 @@ void file::load_elf_header()

 void file::read(Elf64_Off offset, void* data, size_t size)
 {
+// read(fileref, ...) is void, and crashes with assertion failure if  
the

+// file is not long enough. So we need to check first.
+if (::size(_f) < offset + size) {
+throw std::runtime_error("executable too short");
+}
 ::read(_f, data, offset, size);
 }

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv master] osv_execve: fail gracefully if program file does not exist

[Commit Bot]

From: Justin Cinkelj 
Committer: Nadav Har'El 
Branch: master

osv_execve: fail gracefully if program file does not exist

If non-existent path is passed to osv_execve, osv::launch_error is raised.
This terminates VM if nobody catches the exception.

This patch implements exception catching, and silently ignores it -
osv_execve still returns 0 and doesn't set errno if exception occurs.

Signed-off-by: Justin Cinkelj 
Message-Id: <20170818120141.5343-2-justin.cink...@xlab.si>

---
diff --git a/core/osv_execve.cc b/core/osv_execve.cc
--- a/core/osv_execve.cc
+++ b/core/osv_execve.cc
@@ -24,19 +24,34 @@ static int thread_run_app_in_namespace(std::string  
filename,

 {
 const bool new_program = true; // run in new ELF namespace
 long tid = gettid(); // sched::thread::current()->id();
+int ret = -1;
+bool exception_raised = false;

 debugf_execve("thread_run_app_in_namespace... tid=%ld\n", tid);
 *thread_id = tid;

-auto app = osv::application::run_and_join(filename, args, new_program,  
, parent_waiter);

-auto ret = app->get_return_code();
-debugf_execve("thread_run_app_in_namespace ret = %d tid=%ld\n", ret,  
tid);

+try {
+auto app = osv::application::run_and_join(filename, args,  
new_program, , parent_waiter);

+ret = app->get_return_code();
+debugf_execve("thread_run_app_in_namespace ret = %d tid=%ld\n",  
ret, tid);

+}
+catch (osv::launch_error ) {
+exception_raised = true;
+}

 WITH_LOCK(exec_mutex) {
 exited_threads[tid] = ret;
 cond.wake_all();
 }

+if (exception_raised) {
+// osv::application::run_and_join failed, and likely didn't wake  
up parent_waiter.

+// Wake parent now.
+if (parent_waiter) {
+parent_waiter->wake();
+}
+}
+
 // Trigger event notification via file descriptor (fd created with  
eventfd).

 if (notification_fd > 0) {
 uint64_t notif = 1;

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[COMMIT osv-apps master] busybox ash: do not exit if command doesn't exist

[Commit Bot]

From: Justin Cinkelj 
Committer: Nadav Har'El 
Branch: master

busybox ash: do not exit if command doesn't exist

0004 patch prevents poping up spurious error message 'command not found'
when running in ash say 'ls /'. It requires patch to osv_execve to be
applied - osv_execve should clear errno on success, and set errno on
failure.

0005 patch prevent ash exit when non-existent command is tried.

As @nyh suggested, symlink /usr/bin -> /usr/lib is added, so that
ash works without adding /usr/lib to PATH environ.

Signed-off-by: Justin Cinkelj 
Message-Id: <20170818120212.5470-1-justin.cink...@xlab.si>

---
diff --git a/busybox/GET b/busybox/GET
--- a/busybox/GET
+++ b/busybox/GET
@@ -34,7 +34,10 @@ cat < usr.manifest
 /usr/lib/echo: ->/usr/lib/busybox
 /usr/lib/ping: ->/usr/lib/busybox
 #
+# Link /usr/bin (in ash's default PATH) to /usr/lib where we put the  
executables

+/usr/bin: ->/usr/lib
+#
 # test scripts
 ## /**: \${MODULE_DIR}/test-script/**

-EOF
\ No newline at end of file
+EOF
diff --git  
a/busybox/patches/0004-OSv-ash-fix-command-not-found-error-message-when-com.patch  
b/busybox/patches/0004-OSv-ash-fix-command-not-found-error-message-when-com.patch
---  
a/busybox/patches/0004-OSv-ash-fix-command-not-found-error-message-when-com.patch
+++  
b/busybox/patches/0004-OSv-ash-fix-command-not-found-error-message-when-com.patch

@@ -0,0 +1,39 @@
+From 1512e2b29f3d5c19d0735470d74c546bd15ed15a Mon Sep 17 00:00:00 2001
+From: Justin Cinkelj 
+Date: Fri, 18 Aug 2017 07:01:58 +0200
+Subject: [PATCH 4/5] OSv ash: fix command not found error message when  
command

+ was found
+
+Running simple 'ls' in ash showed directory content. But at the end,
+spurious error message 'command not found' was shown. This was due to
+errno not being cleared by osv_execve. This was fixed in osv_execve, now
+it clears errno on success, or sets it to meanigful value (currently
+always ENOENT).
+
+Signed-off-by: Justin Cinkelj 
+---
+ shell/ash.c | 8 
+ 1 file changed, 4 insertions(+), 4 deletions(-)
+
+diff --git a/shell/ash.c b/shell/ash.c
+index bc16394..dc88bd8 100644
+--- a/shell/ash.c
 b/shell/ash.c
+@@ -7731,10 +7731,10 @@ tryexec(IF_FEATURE_SH_STANDALONE(int applet_no,)  
char *cmd, char **argv, char **

+   int ret = 0;
+   pid_t ch_pid = -1;
+   ret = osv_execve(cmd, argv, envp, (long*)_pid, -1);
+-  //errno = 0; // osv_execve should clear errno on success
+-  if(ret)
+-  errno = ENOSYS;
+-  osv_waittid(ch_pid, NULL, 0);
++  // osv_execve now clears errno on success, as expected by 
caller.
++  if(ret == 0) {
++  osv_waittid(ch_pid, NULL, 0);
++  }
+   return;
+   }
+ #endif
+--
+2.9.4
+
diff --git  
a/busybox/patches/0005-OSv-ash-do-not-exit-if-program-to-be-run-fails-to-st.patch  
b/busybox/patches/0005-OSv-ash-do-not-exit-if-program-to-be-run-fails-to-st.patch
---  
a/busybox/patches/0005-OSv-ash-do-not-exit-if-program-to-be-run-fails-to-st.patch
+++  
b/busybox/patches/0005-OSv-ash-do-not-exit-if-program-to-be-run-fails-to-st.patch

@@ -0,0 +1,140 @@
+From 1009e202f7c1214e7ab6185782f1033506a634a8 Mon Sep 17 00:00:00 2001
+From: Justin Cinkelj 
+Date: Fri, 18 Aug 2017 07:26:12 +0200
+Subject: [PATCH 5/5] OSv ash: do not exit if program to be run fails to  
start

+
+If program to be started fails to start (executable file not found), ash
+should not return from its main loop, as this terminates shell.
+
+ash_main interceps exception from raise_exception, and exits because
+exception_type == EXEXIT. The exit is avoided if instead of returning from
+ash_main, code just continues with cmdloop(). Important detail - we have
+to first call popstackmark, and there re-execute setjmp, so that longjmp
+will work.
+
+As EXEXIT is used for both abnormal exit (exception handling, like
+file not found) and normal exit (entering 'exit' cmd into ash), this
+breaks 'exit' cmd. Thus we have to separete the two cases.
+This is done by replacing EXEXIT with:
+ - EXEXIT_CMD, when 'exit' was passed to ash interpreter.
+ - EXEXIT_VAR, for all other reasons.
+
+Signed-off-by: Justin Cinkelj 
+---
+ shell/ash.c | 44 +---
+ 1 file changed, 37 insertions(+), 7 deletions(-)
+
+diff --git a/shell/ash.c b/shell/ash.c
+index dc88bd8..d8111c0 100644
+--- a/shell/ash.c
 b/shell/ash.c
+@@ -324,7 +324,9 @@ struct globals_misc {
+   /* exceptions */
+ #define EXINT 0 /* SIGINT received */
+ #define EXERROR 1   /* a generic error */
+-#define EXEXIT 4/* exit the shell */
++//#define EXEXIT 4/* exit the shell */
++#define EXEXIT_CMD 4/* exit the shell, due to exit command */
++#define EXEXIT_VAR 5/* exit the shell, due to other 

[COMMIT osv master] ramfs: strip ! from usr.manifest filenames

[Commit Bot]

From: Justin Cinkelj 
Committer: Nadav Har'El 
Branch: master

ramfs: strip ! from usr.manifest filenames

usr.manifest can contain lines like "/osv-path: !/host-path", when
"/host-path" is a symlink. We need to remove the ! before searching
host filesystem for the file.

Fixes #879

Signed-off-by: Justin Cinkelj 
Message-Id: <20170608165411.14645-1-justin.cink...@xlab.si>

---
diff --git a/scripts/mkbootfs.py b/scripts/mkbootfs.py
--- a/scripts/mkbootfs.py
+++ b/scripts/mkbootfs.py
@@ -30,6 +30,8 @@ def expand(items):
 yield (name, hostname)

 def unsymlink(f):
+if f.startswith('!'):
+f = f[1:]
 try:
 link = os.readlink(f)
 if link.startswith('/'):

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.