Hi Bram,
2016/8/6 Sat 22:30:05 UTC+9 Bram Moolenaar wrote:
> Ken Takata wrote:
>
> > When process substitution (e.g. "<(cmd)") is used on bash (or zsh), file
> > encoding detection doesn't work, and an empty buffer is opened.
> > E.g.:
> >
> > $ vim -u NONE --cmd "set fencs=cp932,utf-8" <(echo テスト)
> > # Assuming the system encoding is utf-8.
> >
> > Vim will be executed with the command line like:
> >
> > $ vim -u NONE --cmd "set fencs=cp932,utf-8" /dev/fd/63
> >
> > "/dev/fd/63" is a fifo and it is not seekable.
> > The echoed string "テスト" (utf-8) cannot be converted from cp932 to utf-8, so
> > Vim retries with utf-8, but it fails because the input is not seekable.
> >
> > However reading from stdin can detect the encoding. E.g.:
> >
> > $ echo テスト | vim -u NONE --cmd "set fencs=cp932,utf-8" -
> >
> > I think file encoding detection should also work with process substitution.
> > Attached patch fixes the problem.
>
> Thanks. It would be good to have a test for this. The encoding
> detection is important, but I don't think there is a test for it.
>
> Reading from stdin and fifo require starting Vim. Test49 does this, it
> just uses ../vim for that. That doesn't work on Windows though. But
> the fifo won't work on Windows either.
I added a test for this. I also wanted to add a test for stdin on Windows,
but Windows' echo command cannot write Unicode characters to a pipe. So the
test is disabled on Windows.
BTW, I found that RunVim() in shared.vim didn't work as expected. I also
fixed this.
Regards,
Ken Takata
--
--
You received this message from the "vim_dev" maillist.
Do not top-post! Type your reply below the text you are replying to.
For more information, visit http://www.vim.org/maillist.php
---
You received this message because you are subscribed to the Google Groups
"vim_dev" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to vim_dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.
# HG changeset patch
# Parent fe67d6106f686209c616b7dc937457c909b0ee31
diff --git a/src/Makefile b/src/Makefile
--- a/src/Makefile
+++ b/src/Makefile
@@ -2111,6 +2111,7 @@ test_arglist \
test_reltime \
test_ruby \
test_startup \
+ test_startup_utf8 \
test_searchpos \
test_set \
test_sort \
diff --git a/src/buffer.c b/src/buffer.c
--- a/src/buffer.c
+++ b/src/buffer.c
@@ -70,6 +70,64 @@ static char *e_auabort = N_("E855: Autoc
/* Number of times free_buffer() was called. */
static int buf_free_count = 0;
+/* Read data from buffer for retrying. */
+ static int
+read_buffer(
+ int read_stdin, /* read file from stdin, otherwise fifo */
+ exarg_T *eap, /* for forced 'ff' and 'fenc' or NULL */
+ int flags) /* extra flags for readfile() */
+{
+ int retval = OK;
+ linenr_T line_count;
+
+ /*
+ * Read from the buffer which the text is already filled in and append at
+ * the end. This makes it possible to retry when 'fileformat' or
+ * 'fileencoding' was guessed wrong.
+ */
+ line_count = curbuf->b_ml.ml_line_count;
+ retval = readfile(
+ read_stdin ? NULL : curbuf->b_ffname,
+ read_stdin ? NULL : curbuf->b_fname,
+ (linenr_T)line_count, (linenr_T)0, (linenr_T)MAXLNUM, eap,
+ flags | READ_BUFFER);
+ if (retval == OK)
+ {
+ /* Delete the binary lines. */
+ while (--line_count >= 0)
+ ml_delete((linenr_T)1, FALSE);
+ }
+ else
+ {
+ /* Delete the converted lines. */
+ while (curbuf->b_ml.ml_line_count > line_count)
+ ml_delete(line_count, FALSE);
+ }
+ /* Put the cursor on the first line. */
+ curwin->w_cursor.lnum = 1;
+ curwin->w_cursor.col = 0;
+
+ if (read_stdin)
+ {
+ /* Set or reset 'modified' before executing autocommands, so that
+ * it can be changed there. */
+ if (!readonlymode && !bufempty())
+ changed();
+ else if (retval != FAIL)
+ unchanged(curbuf, FALSE);
+
+#ifdef FEAT_AUTOCMD
+# ifdef FEAT_EVAL
+ apply_autocmds_retval(EVENT_STDINREADPOST, NULL, NULL, FALSE,
+ curbuf, &retval);
+# else
+ apply_autocmds(EVENT_STDINREADPOST, NULL, NULL, FALSE, curbuf);
+# endif
+#endif
+ }
+ return retval;
+}
+
/*
* Open current buffer, that is: open the memfile and read the file into
* memory.
@@ -88,6 +146,7 @@ open_buffer(
#ifdef FEAT_SYN_HL
long old_tw = curbuf->b_p_tw;
#endif
+ int read_fifo = FALSE;
/*
* The 'readonly' flag is only set when BF_NEVERLOADED is being reset.
@@ -143,17 +202,42 @@ open_buffer(
)
{
int old_msg_silent = msg_silent;
-
+#ifdef UNIX
+ int save_bin = curbuf->b_p_bin;
+ int perm;
+#endif
#ifdef FEAT_NETBEANS_INTG
int oldFire = netbeansFireChanges;
netbeansFireChanges = 0;
#endif
+#ifdef UNIX
+ perm = mch_getperm(curbuf->b_ffname);
+ if (perm >= 0 && (0
+# ifdef S_ISFIFO
+ || S_ISFIFO(perm)
+# endif
+# ifdef S_ISSOCK
+ || S_ISSOCK(perm)
+# endif
+ ))
+ read_fifo = TRUE;
+ if (read_fifo)
+ curbuf->b_p_bin = TRUE;
+#endif
if (shortmess(SHM_FILEINFO))
msg_silent = 1;
retval = readfile(curbuf->b_ffname, curbuf->b_fname,
(linenr_T)0, (linenr_T)0, (linenr_T)MAXLNUM, eap,
- flags | READ_NEW);
+ flags | READ_NEW | (read_fifo ? READ_FIFO : 0));
+#ifdef UNIX
+ if (read_fifo)
+ {
+ curbuf->b_p_bin = save_bin;
+ if (retval == OK)
+ retval = read_buffer(FALSE, eap, flags);
+ }
+#endif
msg_silent = old_msg_silent;
#ifdef FEAT_NETBEANS_INTG
netbeansFireChanges = oldFire;
@@ -164,8 +248,7 @@ open_buffer(
}
else if (read_stdin)
{
- int save_bin = curbuf->b_p_bin;
- linenr_T line_count;
+ int save_bin = curbuf->b_p_bin;
/*
* First read the text in binary mode into the buffer.
@@ -179,42 +262,7 @@ open_buffer(
flags | (READ_NEW + READ_STDIN));
curbuf->b_p_bin = save_bin;
if (retval == OK)
- {
- line_count = curbuf->b_ml.ml_line_count;
- retval = readfile(NULL, NULL, (linenr_T)line_count,
- (linenr_T)0, (linenr_T)MAXLNUM, eap,
- flags | READ_BUFFER);
- if (retval == OK)
- {
- /* Delete the binary lines. */
- while (--line_count >= 0)
- ml_delete((linenr_T)1, FALSE);
- }
- else
- {
- /* Delete the converted lines. */
- while (curbuf->b_ml.ml_line_count > line_count)
- ml_delete(line_count, FALSE);
- }
- /* Put the cursor on the first line. */
- curwin->w_cursor.lnum = 1;
- curwin->w_cursor.col = 0;
-
- /* Set or reset 'modified' before executing autocommands, so that
- * it can be changed there. */
- if (!readonlymode && !bufempty())
- changed();
- else if (retval != FAIL)
- unchanged(curbuf, FALSE);
-#ifdef FEAT_AUTOCMD
-# ifdef FEAT_EVAL
- apply_autocmds_retval(EVENT_STDINREADPOST, NULL, NULL, FALSE,
- curbuf, &retval);
-# else
- apply_autocmds(EVENT_STDINREADPOST, NULL, NULL, FALSE, curbuf);
-# endif
-#endif
- }
+ retval = read_buffer(TRUE, eap, flags);
}
/* if first time loading this buffer, init b_chartab[] */
@@ -243,7 +291,7 @@ open_buffer(
#endif
)
changed();
- else if (retval != FAIL && !read_stdin)
+ else if (retval != FAIL && !read_stdin && !read_fifo)
unchanged(curbuf, FALSE);
save_file_ff(curbuf); /* keep this fileformat */
diff --git a/src/fileio.c b/src/fileio.c
--- a/src/fileio.c
+++ b/src/fileio.c
@@ -212,6 +212,7 @@ filemess(
* stdin)
* READ_DUMMY read into a dummy buffer (to check if file contents changed)
* READ_KEEP_UNDO don't clear undo info or read it from a file
+ * READ_FIFO read from fifo/socket instead of a file
*
* return FAIL for failure, OK otherwise
*/
@@ -231,6 +232,7 @@ readfile(
int filtering = (flags & READ_FILTER);
int read_stdin = (flags & READ_STDIN);
int read_buffer = (flags & READ_BUFFER);
+ int read_fifo = (flags & READ_FIFO);
int set_options = newfile || read_buffer
|| (eap != NULL && eap->read_edit);
linenr_T read_buf_lnum = 1; /* next line to read from curbuf */
@@ -431,7 +433,7 @@ readfile(
}
}
- if (!read_stdin && !read_buffer)
+ if (!read_stdin && !read_buffer && !read_fifo)
{
#ifdef UNIX
/*
@@ -489,7 +491,7 @@ readfile(
if (check_readonly && !readonlymode)
curbuf->b_p_ro = FALSE;
- if (newfile && !read_stdin && !read_buffer)
+ if (newfile && !read_stdin && !read_buffer && !read_fifo)
{
/* Remember time of file. */
if (mch_stat((char *)fname, &st) >= 0)
@@ -1101,6 +1103,7 @@ retry:
* and we can't do it internally or with iconv().
*/
if (fio_flags == 0 && !read_stdin && !read_buffer && *p_ccv != NUL
+ && !read_fifo
# ifdef USE_ICONV
&& iconv_fd == (iconv_t)-1
# endif
@@ -1149,7 +1152,7 @@ retry:
/* Set "can_retry" when it's possible to rewind the file and try with
* another "fenc" value. It's FALSE when no other "fenc" to try, reading
* stdin or fixed at a specific encoding. */
- can_retry = (*fenc != NUL && !read_stdin && !keep_dest_enc);
+ can_retry = (*fenc != NUL && !read_stdin && !read_fifo && !keep_dest_enc);
#endif
if (!skip_read)
@@ -1166,6 +1169,7 @@ retry:
&& curbuf->b_ffname != NULL
&& curbuf->b_p_udf
&& !filtering
+ && !read_fifo
&& !read_stdin
&& !read_buffer);
if (read_undo_file)
@@ -2666,7 +2670,7 @@ failed:
#endif
#ifdef FEAT_AUTOCMD
- if (!read_stdin && !read_buffer)
+ if (!read_stdin && !read_fifo && (!read_buffer || sfname != NULL))
{
int m = msg_scroll;
int n = msg_scrolled;
@@ -2685,7 +2689,7 @@ failed:
if (filtering)
apply_autocmds_exarg(EVENT_FILTERREADPOST, NULL, sfname,
FALSE, curbuf, eap);
- else if (newfile)
+ else if (newfile || (read_buffer && sfname != NULL))
{
apply_autocmds_exarg(EVENT_BUFREADPOST, NULL, sfname,
FALSE, curbuf, eap);
diff --git a/src/testdir/Make_all.mak b/src/testdir/Make_all.mak
--- a/src/testdir/Make_all.mak
+++ b/src/testdir/Make_all.mak
@@ -185,6 +185,7 @@ NEW_TESTS = test_arglist.res \
test_quickfix.res \
test_ruby.res \
test_startup.res \
+ test_startup_utf8.res \
test_stat.res \
test_syntax.res \
test_textobjects.res \
diff --git a/src/testdir/shared.vim b/src/testdir/shared.vim
--- a/src/testdir/shared.vim
+++ b/src/testdir/shared.vim
@@ -130,7 +130,7 @@ endfunc
" Plugins are not loaded, unless 'loadplugins' is set in "before".
" Return 1 if Vim could be executed.
func RunVim(before, after, arguments)
- call RunVimPiped(a:before, a:after, a:arguments, '')
+ return RunVimPiped(a:before, a:after, a:arguments, '')
endfunc
func RunVimPiped(before, after, arguments, pipecmd)
diff --git a/src/testdir/test_startup_utf8.vim b/src/testdir/test_startup_utf8.vim
new file mode 100644
--- /dev/null
+++ b/src/testdir/test_startup_utf8.vim
@@ -0,0 +1,56 @@
+" Tests for startup using utf-8.
+if !has('multi_byte')
+ finish
+endif
+
+source shared.vim
+
+func Test_read_stdin_utf8()
+ " MS-Windows cannot handle Unicode characters in stdin.
+ if has('win32')
+ return
+ endif
+ let before = [
+ \ 'set enc=utf-8',
+ \ 'set fencs=cp932,utf-8',
+ \ ]
+ let after = [
+ \ 'write ++enc=utf-8 Xtestout',
+ \ 'quit!',
+ \ ]
+ if RunVimPiped(before, after, '-', 'echo テスト€ÀÈÌÒÙ | ')
+ let lines = readfile('Xtestout')
+ " MS-Windows adds a space after the word
+ call assert_equal(['テスト€ÀÈÌÒÙ'], split(lines[0]))
+ endif
+ call delete('Xtestout')
+endfunc
+
+func Test_read_fifo_utf8()
+ if !has('unix')
+ return
+ endif
+ " Using bash/zsh's process substitution.
+ if executable('bash')
+ set shell=bash
+ elseif executable('zsh')
+ set shell=zsh
+ else
+ return
+ endif
+ let before = [
+ \ 'set enc=utf-8',
+ \ 'set fencs=cp932,utf-8',
+ \ ]
+ let after = [
+ \ 'write ++enc=utf-8 Xtestout',
+ \ 'quit!',
+ \ ]
+ if RunVim(before, after, '<(echo テスト€ÀÈÌÒÙ)')
+ let lines = readfile('Xtestout')
+ call assert_equal(['テスト€ÀÈÌÒÙ'], lines)
+ else
+ call assert_equal('', 'RunVim failed.')
+ endif
+ call delete('Xtestout')
+endfunc
diff --git a/src/vim.h b/src/vim.h
--- a/src/vim.h
+++ b/src/vim.h
@@ -989,7 +989,8 @@ extern char *(*dyn_libintl_textdomain)(c
#define READ_STDIN 0x04 /* read from stdin */
#define READ_BUFFER 0x08 /* read from curbuf (converting stdin) */
#define READ_DUMMY 0x10 /* reading into a dummy buffer */
-#define READ_KEEP_UNDO 0x20 /* keep undo info*/
+#define READ_KEEP_UNDO 0x20 /* keep undo info */
+#define READ_FIFO 0x40 /* read from fifo or socket */
/* Values for change_indent() */
#define INDENT_SET 1 /* set indent */
