Merge branch 'master' into gnuplot

lisp/langs/org-babel-R.el

@@ -45,8 +45,9 @@ called by `org-babel-execute-src-block' via multiple-value-bind."
 		       (lambda (pair)
 			 (org-babel-R-assign-elisp (car pair) (cdr pair)))
 		       vars "\n") "\n" body "\n"))
-	  (session (org-babel-R-initiate-session session)))
-      (org-babel-R-evaluate session full-body result-type))))
+	  (session (org-babel-R-initiate-session session))
+	  (column-names-p (cdr (assoc :colnames params))))
+      (org-babel-R-evaluate session full-body result-type column-names-p))))
 
 (defun org-babel-prep-session:R (session params)
   "Prepare SESSION according to the header arguments specified in PARAMS."
@@ -71,8 +72,8 @@ called by `org-babel-execute-src-block' via multiple-value-bind."
         (with-temp-file transition-file
           (insert (orgtbl-to-tsv value '(:fmt org-babel-R-quote-tsv-field)))
           (insert "\n"))
-        (format "%s <- read.table(\"%s\", header=FALSE, sep=\"\\t\", as.is=TRUE)"
-                name transition-file))
+        (format "%s <- read.table(\"%s\", header=%s, sep=\"\\t\", as.is=TRUE)"
+                name transition-file (if (eq (second value) 'hline) "TRUE" "FALSE")))
     (format "%s <- %s" name (org-babel-R-quote-tsv-field value))))
 
 (defun org-babel-R-initiate-session (session)
@@ -89,9 +90,9 @@ called by `org-babel-execute-src-block' via multiple-value-bind."
 (defvar org-babel-R-eoe-indicator "'org_babel_R_eoe'")
 (defvar org-babel-R-eoe-output "[1] \"org_babel_R_eoe\"")
 (defvar org-babel-R-wrapper-method "main <- function ()\n{\n%s\n}
-write.table(main(), file=\"%s\", sep=\"\\t\", na=\"nil\",row.names=FALSE, col.names=FALSE, quote=FALSE)")
+write.table(main(), file=\"%s\", sep=\"\\t\", na=\"nil\",row.names=FALSE, col.names=%s, quote=FALSE)")
 
-(defun org-babel-R-evaluate (buffer body result-type)
+(defun org-babel-R-evaluate (buffer body result-type colnames)
   "Pass BODY to the R process in BUFFER.  If RESULT-TYPE equals
 'output then return a list of the outputs of the statements in
 BODY, if RESULT-TYPE equals 'value then return the value of the
@@ -108,15 +109,16 @@ last statement in BODY, as elisp."
 	   (with-temp-buffer (insert-file-contents out-tmp-file) (buffer-string)))
           (value
            (with-temp-file in-tmp-file
-             (insert (format org-babel-R-wrapper-method body out-tmp-file)))
+             (insert (format org-babel-R-wrapper-method
+			     body out-tmp-file (if column-names-p "TRUE" "FALSE"))))
            (shell-command (format "R --no-save < '%s'" in-tmp-file))
-	   (org-babel-import-elisp-from-file out-tmp-file))))
+	   (org-babel-R-process-value-result
+	    (org-babel-import-elisp-from-file out-tmp-file) column-names-p))))
     ;; comint session evaluation
     (org-babel-comint-in-buffer buffer
       (let* ((tmp-file (make-temp-file "org-babel-R"))
              (last-value-eval
-              (format "write.table(.Last.value, file=\"%s\", sep=\"\\t\", na=\"nil\",row.names=FALSE, col.names=FALSE, quote=FALSE)"
-                      tmp-file))
+              (format "write.table(.Last.value, file=\"%s\", sep=\"\\t\", na=\"nil\",row.names=FALSE, col.names=%s, quote=FALSE)" tmp-file (if column-names-p "TRUE" "FALSE")))
              (full-body (mapconcat #'org-babel-chomp
 				   (list body last-value-eval org-babel-R-eoe-indicator) "\n"))
              (raw (org-babel-comint-with-output buffer org-babel-R-eoe-output nil
@@ -137,9 +139,18 @@ last statement in BODY, as elisp."
 				 el))))
 			 (mapcar #'org-babel-trim raw))))))
         (case result-type
-          (output (org-babel-trim (mapconcat #'identity results "\n")))
-          (value (org-babel-import-elisp-from-file tmp-file)))))))
-
+          (output (org-babel-chomp (mapconcat #'identity results "\n")))
+          (value (org-babel-R-process-value-result
+		  (org-babel-import-elisp-from-file tmp-file) column-names-p)))))))
+
+(defun org-babel-R-process-value-result (result column-names-p)
+  "R-specific processing of return value prior to return to org-babel.
+
+Currently, insert hline if column names in output have been requested."
+  (if column-names-p
+      (cons (car result) (cons 'hline (cdr result)))
+    result))
+  
 
 (provide 'org-babel-R)
 ;;; org-babel-R.el ends here

lisp/org-babel.el

@@ -578,14 +578,14 @@ the table is trivial, then return it as a scalar."
             (setq result (mapcar (lambda (row)
                                    (mapcar #'org-babel-string-read row))
                                  (org-table-to-lisp))))
-        (error nil))
-      (if (null (cdr result)) ;; if result is trivial vector, then scalarize it
-          (if (consp (car result))
-              (if (null (cdr (car result)))
-                  (caar result)
-                result)
-            (car result))
-        result))))
+        (error nil)))
+    (if (null (cdr result)) ;; if result is trivial vector, then scalarize it
+	(if (consp (car result))
+	    (if (null (cdr (car result)))
+		(caar result)
+	      result)
+	  (car result))
+      result)))
 
 (defun org-babel-string-read (cell)
   "Strip nested \"s from around strings in exported R values."

org-babel.org

@@ -123,11 +123,14 @@ rand(100)
 #+end_src
 
 #+srcname: show-distribution
-#+begin_src R :var grades=grades-table
-hist(grades[,2])
+#+begin_src R :var grades=grades-table :session *R*
+hist(grades$grade)
 #+end_src
 
 
+
+
+
 ** communication between people
 Quick overview of Org-Mode's exportation abilities, with links to the
 online Org-Mode documentation, a focus on source-code blocks, and the
@@ -204,7 +207,7 @@ would then be [[#sandbox][the sandbox]].
 #+end_src
 
 
-* Tasks [32/51]
+* Tasks [32/50]
 ** PROPOSED optional timestamp for output
    Add option to place an (inactive) timestamp at the #+resname, to
    record when that output was generated.
@@ -367,6 +370,57 @@ languages which almost always produce graphical output should set
 results in the buffer.  Then if there is a combination of =silent= and
 =file= =:results= headers we could drop the results to a temp buffer
 and pop open that buffer...
+
+*** TODO R graphics to screen means session evaluation
+    If R graphical output is going to screen then evaluation must be
+    in a session, otherwise the graphics will disappear as soon as the
+    R process dies.
+
+*** Adding to a discussion started in email
+    I'm not deeply wedded to these ideas, just noting them down. I'm
+    probably just thinking of R and haven't really thought about how
+    this fits with the other graphics-generating languages.
+Dan:
+> I used the approach below to get graphical file output
+> today, which is one idea at least. Maybe it could be linked up with
+> your :results file variable. (Or do we need a :results image for R?)
+>
+Eric:
+I don't think we need a special image results variable, but I may be
+missing what the code below accomplishes.  Would the task I added about
+adding org-open-at-point functionality to source code blocks take care
+of this need?
+
+Dan: I'm not sure. I think the ability for a script to generate both
+text and graphical output might be a natural expectation, at least for
+R users.
+
+>
+> Dan
+>
+> #+srcname: cohort-scatter-plots-2d(org_babel_graphical_output_file="cohort-scatter-plots-2d.png")
+> #+begin_src R 
+>   if(exists("org_babel_output_file"))
+>       png(filename=org_babel_graphical_output_file, width=1000, height=1000)
+>   ## plotting code in here
+>   if(exists("org_babel_graphical_output_file")) dev.off()
+> #+end_src
+
+Dan: Yes, the results :file option is nice for dealing with graphical
+output, and that could well be enough. Something based on the scheme
+above would have a couple of points in its favour:
+1. It's easy to switch between output going to on-screen graphics and
+   output going to file: Output will go to screen unless a string variable
+   with a standard name (e.g. ""org_babel_graphical_output_file"")
+   exists in which case it will go to the file indicated by the value
+   of that variable.
+2. The block can return a result / script output, as well as produce
+   graphical output.
+
+In interactive use we might want to allow the user to choose between
+screen and file output. In non-interactive use such as export, it
+would be file output (subject to the :exports directives).
+
 ** TODO =\C-c \C-o= to open results of source block
 by adding a =defadvice= to =org-open-at-point= we can use the common
 =\C-c \C-o= keybinding to open the results of a source-code block.
@@ -386,20 +440,21 @@ there are spaces in the message and scalar otherwise.
 
 [Not any more]
 
-#+begin_src R :var msg=msg-from-python
+#+srcname: msg-from-R(msg=msg-from-python)
+#+begin_src R
 paste(msg, "und R", sep=" ")
 #+end_src
 
 #+resname:
 : org-babel speaks elisp y python und R
 
-#+srcname: msg-from-python
-#+begin_src python :var msg=msg-from-elisp
+#+srcname: msg-from-python(msg=msg-from-elisp)
+#+begin_src python
 msg + " y python"
 #+end_src
 
-#+srcname: msg-from-elisp
-#+begin_src emacs-lisp :var msg="org-babel speaks"
+#+srcname: msg-from-elisp(msg="org-babel speaks")
+#+begin_src emacs-lisp
 (concat msg " elisp")
 #+end_src
 ** STARTED share org-babel [1/4]
@@ -477,10 +532,22 @@ we should color these blocks differently
 *** TODO refine html exportation
 should use a span class, and should show original source in tool-tip
 
-** TODO formulate general rules for handling vectors and tables / matrices with names
-   This is non-trivial, but may be worth doing, in particular to
-   develop a nice framework for sending data to/from R.
-*** Notes
+** STARTED Column (and row) names of tables in R input/output
+*** DONE column names
+    This has been implemented: Automatic on input to R; optional in output.
+*** TODO row names
+    Perhaps add a :rownames header arg. This would be an integer
+    (usually 1) which would have the effect of post-processing all the
+    variables created in the R session in the following way: if the
+    integer is j, set the row names to the contents of column j and
+    delete column j. Perhaps it is artificial to allow this integer to
+    take any value other than 1. The default would be nil which would
+    mean no such behaviour.
+*** Old notes
+    [I don't think it's as problematic as this makes out]
+    This is non-trivial, but may be worth doing, in particular to
+    develop a nice framework for sending data to/from R.
+**** Notes
     In R, indexing vector elements, and rows and columns, using
     strings rather than integers is an important part of the
     language.
@@ -541,7 +608,7 @@ g    7   20
    The full org table functionality exeplified [[http://orgmode.org/manual/Advanced-features.html#Advanced-features][here]] has features that
    we would not support in e.g. R (like names for the row below).
    
-*** Initial statement: allow tables with hline to be passed as args into R
+**** Initial statement: allow tables with hline to be passed as args into R
    This doesn't seem to work at the moment (example below). It would
    also be nice to have a natural way for the column names of the org
    table to become the column names of the R data frame, and to have
@@ -560,7 +627,7 @@ g    7   20
 | 1 |         2 | 3 |
 | 4 | schulte   | 6 |
 
-#+begin_src R var tabel=egtable
+#+begin_src R :var tabel=egtable :colnames t
 tabel
 #+end_src
 
@@ -629,23 +696,6 @@ Another example is in the [[*operations%20in%20on%20tables][grades example]].
 *** Current design
     This is covered by the [[file:library-of-babel.org][Library of Babel]], which will contain
     ready-made source blocks designed to carry out useful common tasks.
-** PROPOSED Are we happy with current behaviour regarding vector/scalar output?
-This simple example of multilingual chaining produces vector output if
-there are spaces in the message and scalar otherwise.
-
-#+begin_src R :var msg=msg-from-python
-paste(msg, "und_R", sep="_")
-#+end_src
-
-#+srcname: msg-from-python
-#+begin_src python :var msg=msg-from-elisp
-msg + "_y_python"
-#+end_src
-
-#+srcname: msg-from-elisp
-#+begin_src emacs-lisp :var msg="org-babel_speaks"
-(concat msg "_elisp")
-#+end_src
 
 ** PROPOSED conversion between org-babel and noweb (e.g. .Rnw) format
    I haven't thought about this properly. Just noting it down. What
@@ -723,67 +773,11 @@ for quick tests
 mean(mean(vec))
 #+end_src
 
+#+resname:
 : 2
 
-** DEFERRED re-implement R evaluation using ess-command or ess-execute
-   I don't have any complaints with the current R evaluation code or
-   behaviour, but I think it would be good to use the ESS functions
-   from a political point of view. Plus of course it has the normal
-   benefits of an API (insulates us from any underlying changes etc). [DED]
-
-   I'll look into this.  I believe that I looked at and rejected these
-   functions initially but now I can't remember why.  I agree with
-   your overall point about using API's where available.  I will take
-   a look back at these and either switch to using the ess commands,
-   or at least articulate under this TODO the reasons for using our
-   custom R-interaction commands. [Eric]
-
-   ess-execute
 
-   Lets just replace =org-babel-R-input-command= with =ess-execute=.
-
-   I tried this, and although it works in some situations, I find that
-   =ess-command= will often just hang indefinitely without returning
-   results.  Also =ess-execute= will occasionally hang, and pops up
-   the buffer containing the results of the command's execution, which
-   is undesirable.  For now these functions can not be used.  Maybe
-   someone more familiar with the ESS code can recommend proper usage
-   of =ess-command= or some other lower-level function which could be
-   used in place of [[file:lisp/org-babel-R.el::defun%20org-babel%20R%20input%20command%20command][org-babel-R-input-command]].
-
-*** ess functions
-   
-#+begin_quote ess-command
-(ess-command COM &optional BUF SLEEP NO-PROMPT-CHECK)
-
-Send the ESS process command COM and delete the output
-from the ESS process buffer.  If an optional second argument BUF exists
-save the output in that buffer. BUF is erased before use.
-COM should have a terminating newline.
-Guarantees that the value of .Last.value will be preserved.
-When optional third arg SLEEP is non-nil, `(sleep-for (* a SLEEP))'
-will be used in a few places where `a' is proportional to `ess-cmd-delay'.
-#+end_quote
-
-#+begin_quote ess-execute
-(ess-execute COMMAND &optional INVERT BUFF MESSAGE)
-
-Send a command to the ESS process.
-A newline is automatically added to COMMAND.  Prefix arg (or second arg
-INVERT) means invert the meaning of
-`ess-execute-in-process-buffer'.  If INVERT is 'buffer, output is
-forced to go to the process buffer.  If the output is going to a
-buffer, name it *BUFF*.	 This buffer is erased before use.  Optional
-fourth arg MESSAGE is text to print at the top of the buffer (defaults
-to the command if BUFF is not given.)
-#+end_quote
-
-*** out current setup
-
-    1) The body of the R source code block is wrapped in a function
-    2) The function is called inside of a =write.table= function call
-       writing the results to a table
-    3) The table is read using =org-table-import=
+: 2
 
 ** DEFERRED Rework Interaction with Running Processes [2/5]
 *** DONE robust to errors interrupting execution
@@ -879,6 +873,65 @@ $0
 
 waiting for guidance from those more familiar with yasnippets
 
+** REJECTED re-implement R evaluation using ess-command or ess-execute
+   I don't have any complaints with the current R evaluation code or
+   behaviour, but I think it would be good to use the ESS functions
+   from a political point of view. Plus of course it has the normal
+   benefits of an API (insulates us from any underlying changes etc). [DED]
+
+   I'll look into this.  I believe that I looked at and rejected these
+   functions initially but now I can't remember why.  I agree with
+   your overall point about using API's where available.  I will take
+   a look back at these and either switch to using the ess commands,
+   or at least articulate under this TODO the reasons for using our
+   custom R-interaction commands. [Eric]
+
+   ess-execute
+
+   Lets just replace =org-babel-R-input-command= with =ess-execute=.
+
+   I tried this, and although it works in some situations, I find that
+   =ess-command= will often just hang indefinitely without returning
+   results.  Also =ess-execute= will occasionally hang, and pops up
+   the buffer containing the results of the command's execution, which
+   is undesirable.  For now these functions can not be used.  Maybe
+   someone more familiar with the ESS code can recommend proper usage
+   of =ess-command= or some other lower-level function which could be
+   used in place of [[file:lisp/org-babel-R.el::defun%20org-babel%20R%20input%20command%20command][org-babel-R-input-command]].
+
+*** ess functions
+   
+#+begin_quote ess-command
+(ess-command COM &optional BUF SLEEP NO-PROMPT-CHECK)
+
+Send the ESS process command COM and delete the output
+from the ESS process buffer.  If an optional second argument BUF exists
+save the output in that buffer. BUF is erased before use.
+COM should have a terminating newline.
+Guarantees that the value of .Last.value will be preserved.
+When optional third arg SLEEP is non-nil, `(sleep-for (* a SLEEP))'
+will be used in a few places where `a' is proportional to `ess-cmd-delay'.
+#+end_quote
+
+#+begin_quote ess-execute
+(ess-execute COMMAND &optional INVERT BUFF MESSAGE)
+
+Send a command to the ESS process.
+A newline is automatically added to COMMAND.  Prefix arg (or second arg
+INVERT) means invert the meaning of
+`ess-execute-in-process-buffer'.  If INVERT is 'buffer, output is
+forced to go to the process buffer.  If the output is going to a
+buffer, name it *BUFF*.	 This buffer is erased before use.  Optional
+fourth arg MESSAGE is text to print at the top of the buffer (defaults
+to the command if BUFF is not given.)
+#+end_quote
+
+*** out current setup
+
+    1) The body of the R source code block is wrapped in a function
+    2) The function is called inside of a =write.table= function call
+       writing the results to a table
+    3) The table is read using =org-table-import=
 ** DONE use example block for large amounts of stdout output?
    We're currently `examplizing' with : at the beginning of the line,
    but should larger amounts of output be in a
@@ -2088,7 +2141,7 @@ into custom gnuplot header arguments
 (see [[* file result types][file result types]])
 
 
-* Bugs [21/34]
+* Bugs [22/34]
 ** TODO avoid stripping whitespace from output when :results output
 ** TODO problem with newlines in output when :results value
 #+begin_src python :results value
@@ -2169,11 +2222,6 @@ the same for the other languages. [Dan]
 ** TODO are the org-babel-trim s necessary?
    at the end of e.g. org-babel-R-evaluate, org-babel-python-evaluate, but
    not org-babel-ruby-evaluate
-** TODO elisp reference fails for literal number
-#+srcname: elisp-test(a=4)
-#+begin_src emacs-lisp 
-(message a)
-#+end_src
 ** TODO use new merge function [[file:lisp/org-babel-ref.el::t%20nil%20org%20combine%20plists%20args%20nil][here]]?
    And at other occurrences of org-combine-plists?
 ** TODO creeping blank lines
@@ -2615,6 +2663,12 @@ recognition of ruby arrays as such.
 #+end_src
 
 | 1 | 2 | 3 | 4 |
+** REJECTED elisp reference fails for literal number
+   That's a bug in Dan's elisp, not in org-babel.
+#+srcname: elisp-test(a=4)
+#+begin_src emacs-lisp 
+(message a)
+#+end_src
 
 
 
@@ -2645,6 +2699,8 @@ of these tests may fail.
 | ruby                    | table-ruby                 |     |       1-2-3 |       1-2-3 | pass |
 | python                  | table-python               |     |           5 |           5 | pass |
 | R                       | table-R                    |     |         3.5 |         3.5 | pass |
+| R: col names in R       | table-R-colnames           |     |          -3 |          -3 | pass |
+| R: col names in org     | table-R-colnames-org       |     |         169 |         169 | pass |
 |-------------------------+----------------------------+-----+-------------+-------------+------|
 | source block references |                            |     |             |             | pass |
 |-------------------------+----------------------------+-----+-------------+-------------+------|
@@ -2678,7 +2734,8 @@ of these tests may fail.
 #+TBLFM: $5='(if (= (length $3) 1) (progn (message (format "running %S" '(sbe $2 (n $3)))) (sbe $2 (n $3))) (sbe $2))::$6='(if (string= $4 $5) "pass" (format "expected %S but was %S" $4 $5))
 #+TBLFM: $5=""::$6=""
 
-The second line (followed by replacing '[]' with '') can be used to blank out the table results, in the absence of a better method.
+The second TBLFM line (followed by replacing '[]' with '') can be used
+to blank out the table results, in the absence of a better method.
 
 ** basic tests
 
@@ -2724,6 +2781,12 @@ b + 4
 | 1 | 2 | 3 |
 | 4 | 5 | 6 |
 
+#+tblname: test-table-colnames
+| var1 | var2 | var3 |
+|------+------+------|
+|    1 |   22 |   13 |
+|   41 |   55 |   67 |
+
 #+srcname: table-elisp
 #+begin_src emacs-lisp :results silent :var table=test-table
 (length (car table))
@@ -2741,12 +2804,34 @@ table.first.join("-")
 table[1][1]
 #+end_src
 
-
-#+srcname: table-R
-#+begin_src R :var table=test-table
+#+srcname: table-R(table=test-table)
+#+begin_src R
 mean(mean(table))
 #+end_src
 
+#+srcname: table-R-colnames(table=test-table-colnames)
+#+begin_src R :results silent
+sum(table$var2 - table$var3)
+#+end_src
+
+#+srcname: R-square(x=default-name-doesnt-exist)
+#+begin_src R :colnames t
+x^2
+#+end_src
+
+This should return 169. The fact that R is able to use the column name
+to index the data frame (x$var3) proves that a table with column names
+(a header row) has been recognised as input for the R-square function
+block, and that the R-square block has output an elisp table with
+column names, and that the colnames have again been recognised when
+creating the R variables in this block.
+#+srcname: table-R-colnames-org(x = R-square(x=test-table-colnames))
+#+begin_src R
+x$var3[1]
+#+end_src
+
+
+
 
 ** references