%!PS-Adobe-3.0
%%Title: (Integrative.journal)
%%Creator: (Microsoft Word: LaserWriter 8 B1-8.1.1)
%%CreationDate: (12:30 pm Monday, July 1, 1996)
%%For: (steve)
%%Pages: 29
%%DocumentFonts: Times-Roman Times-Bold Times-Italic Palatino-Roman Helvetica
%%DocumentNeededFonts: Times-Roman Times-Bold Times-Italic Palatino-Roman Helvetica
%%DocumentSuppliedFonts:
%%DocumentData: Clean7Bit
%%PageOrder: Ascend
%%Orientation: Portrait
%ADO_PaperArea: -31 -31 761 581
%ADO_ImageableArea: 0 0 730 552
%%EndComments
/md 203 dict def md begin
/currentpacking where {pop /sc_oldpacking currentpacking def true setpacking}if
%%BeginFile: adobe_psp_errorhandler
%%Copyright: Copyright 1985-1993 Adobe Systems Incorporated. All Rights Reserved.
/currentpacking where
{pop/oldpack currentpacking def/setpacking where
{
pop false setpacking
}if
}if
/$brkpage 64 dict def $brkpage begin
/prnt
{dup type/stringtype ne{=string cvs}if dup length 6 mul/tx exch def/ty 10 def
currentpoint/toy exch def/tox exch def 1 setgray newpath
tox toy 2 sub moveto 0 ty rlineto tx 0 rlineto 0 ty neg rlineto
closepath fill tox toy moveto 0 setgray show}bind def
/nl{currentpoint exch pop lmargin exch moveto 0 -10 rmoveto}def
/=={/cp 0 def typeprint nl}def
/typeprint{dup type exec}readonly def
/lmargin 72 def
/rmargin 72 def
/tprint
{dup length cp add rmargin gt{nl/cp 0 def}if
dup length cp add/cp exch def prnt}readonly def
/cvsprint{=string cvs tprint( )tprint}readonly def
/integertype{cvsprint}readonly def
/realtype{cvsprint}readonly def
/booleantype{cvsprint}readonly def
/operatortype{(--)tprint =string cvs tprint(-- )tprint}readonly def
/marktype{pop(-mark- )tprint}readonly def
/dicttype{pop(-dictionary- )tprint}readonly def
/nulltype{pop(-null- )tprint}readonly def
/filetype{pop(-filestream- )tprint}readonly def
/savetype{pop(-savelevel- )tprint}readonly def
/fonttype{pop(-fontid- )tprint}readonly def
/nametype{dup xcheck not{(/)tprint}if cvsprint}readonly def
/stringtype
{dup rcheck{(\()tprint tprint(\))tprint}{pop(-string- )tprint}ifelse
}readonly def
/arraytype
{dup rcheck{dup xcheck
{({)tprint{typeprint}forall(})tprint}
{([)tprint{typeprint}forall(])tprint}ifelse}{pop(-array- )tprint}ifelse
}readonly def
/packedarraytype
{dup rcheck{dup xcheck
{({)tprint{typeprint}forall(})tprint}
{([)tprint{typeprint}forall(])tprint}ifelse}{pop(-packedarray- )tprint}ifelse
}readonly def
/courier/Courier findfont 10 scalefont def
end
errordict/handleerror
{systemdict begin $error begin $brkpage begin newerror
{/newerror false store
vmstatus pop pop 0 ne{grestoreall}if initgraphics courier setfont
lmargin 720 moveto(ERROR: )prnt errorname prnt
nl(OFFENDING COMMAND: )prnt/command load prnt
$error/ostack known{
$error/ostack get type dup/arraytype eq exch/packedarraytype eq or
{nl nl(STACK:)prnt nl nl $error/ostack get aload length{==}repeat}if}if
systemdict/showpage get exec(%%[ Error: )print
errorname =print(; OffendingCommand: )print/command
load =print( ]%%)= flush}if end end end}
dup 0 systemdict put dup 4 $brkpage put bind readonly put
/currentpacking where
{pop/setpacking where
{
pop oldpack setpacking
}if
}if
%%EndFile
%%BeginFile: adobe_psp_basic
%%Copyright: Copyright 1990-1993 Adobe Systems Incorporated. All Rights Reserved.
/bd{bind def}bind def
/xdf{exch def}bd
/xs{exch store}bd
/ld{load def}bd
/Z{0 def}bd
/T/true
/F/false
/:L/lineto
/lw/setlinewidth
/:M/moveto
/rl/rlineto
/rm/rmoveto
/:C/curveto
/:T/translate
/:K/closepath
/:mf/makefont
/gS/gsave
/gR/grestore
/np/newpath
14{ld}repeat
/$m matrix def
/av 81 def
/por true def
/normland false def
/psb-nosave{}bd
/pse-nosave{}bd
/us Z
/psb{/us save store}bd
/pse{us restore}bd
/level2
/languagelevel where
{
pop languagelevel 2 ge
}{
false
}ifelse
def
/featurecleanup
{
stopped
cleartomark
countdictstack exch sub dup 0 gt
{
{end}repeat
}{
pop
}ifelse
}bd
/noload Z
/startnoload
{
{/noload save store}if
}bd
/endnoload
{
{noload restore}if
}bd
level2 startnoload
/setjob
{
statusdict/jobname 3 -1 roll put
}bd
/setcopies
{
userdict/#copies 3 -1 roll put
}bd
level2 endnoload level2 not startnoload
/setjob
{
1 dict begin/JobName xdf currentdict end setuserparams
}bd
/setcopies
{
1 dict begin/NumCopies xdf currentdict end setpagedevice
}bd
level2 not endnoload
/pm Z
/mT Z
/sD Z
/realshowpage Z
/initializepage
{
/pm save store mT concat
}bd
/endp
{
pm restore showpage
}def
/$c/DeviceRGB def
/rectclip where
{
pop/rC/rectclip ld
}{
/rC
{
np 4 2 roll
:M
1 index 0 rl
0 exch rl
neg 0 rl
:K
clip np
}bd
}ifelse
/rectfill where
{
pop/rF/rectfill ld
}{
/rF
{
gS
np
4 2 roll
:M
1 index 0 rl
0 exch rl
neg 0 rl
fill
gR
}bd
}ifelse
/rectstroke where
{
pop/rS/rectstroke ld
}{
/rS
{
gS
np
4 2 roll
:M
1 index 0 rl
0 exch rl
neg 0 rl
:K
stroke
gR
}bd
}ifelse
%%EndFile
%%BeginFile: adobe_psp_colorspace_level1
%%Copyright: Copyright 1991-1993 Adobe Systems Incorporated. All Rights Reserved.
/G/setgray ld
/:F/setrgbcolor ld
%%EndFile
%%BeginFile: adobe_psp_uniform_graphics
%%Copyright: Copyright 1990-1993 Adobe Systems Incorporated. All Rights Reserved.
/@a
{
np :M 0 rl :L 0 exch rl 0 rl :L fill
}bd
/@b
{
np :M 0 rl 0 exch rl :L 0 rl 0 exch rl fill
}bd
/arct where
{
pop
}{
/arct
{
arcto pop pop pop pop
}bd
}ifelse
/x1 Z
/x2 Z
/y1 Z
/y2 Z
/rad Z
/@q
{
/rad xs
/y2 xs
/x2 xs
/y1 xs
/x1 xs
np
x2 x1 add 2 div y1 :M
x2 y1 x2 y2 rad arct
x2 y2 x1 y2 rad arct
x1 y2 x1 y1 rad arct
x1 y1 x2 y1 rad arct
fill
}bd
/@s
{
/rad xs
/y2 xs
/x2 xs
/y1 xs
/x1 xs
np
x2 x1 add 2 div y1 :M
x2 y1 x2 y2 rad arct
x2 y2 x1 y2 rad arct
x1 y2 x1 y1 rad arct
x1 y1 x2 y1 rad arct
:K
stroke
}bd
/@i
{
np 0 360 arc fill
}bd
/@j
{
gS
np
:T
scale
0 0 .5 0 360 arc
fill
gR
}bd
/@e
{
np
0 360 arc
:K
stroke
}bd
/@f
{
np
$m currentmatrix
pop
:T
scale
0 0 .5 0 360 arc
:K
$m setmatrix
stroke
}bd
/@k
{
gS
np
:T
0 0 :M
0 0 5 2 roll
arc fill
gR
}bd
/@l
{
gS
np
:T
0 0 :M
scale
0 0 .5 5 -2 roll arc
fill
gR
}bd
/@m
{
np
arc
stroke
}bd
/@n
{
np
$m currentmatrix
pop
:T
scale
0 0 .5 5 -2 roll arc
$m setmatrix
stroke
}bd
%%EndFile
%%BeginFile: adobe_psp_customps
%%Copyright: Copyright 1990-1993 Adobe Systems Incorporated. All Rights Reserved.
/$t Z
/$p Z
/$s Z
/$o 1. def
/2state? false def
/ps Z
level2 startnoload
/pushcolor/currentrgbcolor ld
/popcolor/setrgbcolor ld
/setcmykcolor where
{
pop/currentcmykcolor where
{
pop/pushcolor/currentcmykcolor ld
/popcolor/setcmykcolor ld
}if
}if
level2 endnoload level2 not startnoload
/pushcolor
{
currentcolorspace $c eq
{
currentcolor currentcolorspace true
}{
currentcmykcolor false
}ifelse
}bd
/popcolor
{
{
setcolorspace setcolor
}{
setcmykcolor
}ifelse
}bd
level2 not endnoload
/pushstatic
{
ps
2state?
$o
$t
$p
$s
}bd
/popstatic
{
/$s xs
/$p xs
/$t xs
/$o xs
/2state? xs
/ps xs
}bd
/pushgstate
{
save errordict/nocurrentpoint{pop 0 0}put
currentpoint
3 -1 roll restore
pushcolor
currentlinewidth
currentlinecap
currentlinejoin
currentdash exch aload length
np clippath pathbbox
$m currentmatrix aload pop
}bd
/popgstate
{
$m astore setmatrix
2 index sub exch
3 index sub exch
rC
array astore exch setdash
setlinejoin
setlinecap
lw
popcolor
np :M
}bd
/bu
{
pushgstate
gR
pushgstate
2state?
{
gR
pushgstate
}if
pushstatic
pm restore
mT concat
}bd
/bn
{
/pm save store
popstatic
popgstate
gS
popgstate
2state?
{
gS
popgstate
}if
}bd
/cpat{pop 64 div G 8{pop}repeat}bd
%%EndFile
%%BeginFile: adobe_psp_basic_text
%%Copyright: Copyright 1990-1993 Adobe Systems Incorporated. All Rights Reserved.
/S/show ld
/A{
0.0 exch ashow
}bd
/R{
0.0 exch 32 exch widthshow
}bd
/W{
0.0 3 1 roll widthshow
}bd
/J{
0.0 32 4 2 roll 0.0 exch awidthshow
}bd
/V{
0.0 4 1 roll 0.0 exch awidthshow
}bd
/fcflg true def
/fc{
fcflg{
vmstatus exch sub 50000 lt{
(%%[ Warning: Running out of memory ]%%\r)print flush/fcflg false store
}if pop
}if
}bd
/$f[1 0 0 -1 0 0]def
/:ff{$f :mf}bd
/MacEncoding StandardEncoding 256 array copy def
MacEncoding 39/quotesingle put
MacEncoding 96/grave put
/Adieresis/Aring/Ccedilla/Eacute/Ntilde/Odieresis/Udieresis/aacute
/agrave/acircumflex/adieresis/atilde/aring/ccedilla/eacute/egrave
/ecircumflex/edieresis/iacute/igrave/icircumflex/idieresis/ntilde/oacute
/ograve/ocircumflex/odieresis/otilde/uacute/ugrave/ucircumflex/udieresis
/dagger/degree/cent/sterling/section/bullet/paragraph/germandbls
/registered/copyright/trademark/acute/dieresis/notequal/AE/Oslash
/infinity/plusminus/lessequal/greaterequal/yen/mu/partialdiff/summation
/product/pi/integral/ordfeminine/ordmasculine/Omega/ae/oslash
/questiondown/exclamdown/logicalnot/radical/florin/approxequal/Delta/guillemotleft
/guillemotright/ellipsis/space/Agrave/Atilde/Otilde/OE/oe
/endash/emdash/quotedblleft/quotedblright/quoteleft/quoteright/divide/lozenge
/ydieresis/Ydieresis/fraction/currency/guilsinglleft/guilsinglright/fi/fl
/daggerdbl/periodcentered/quotesinglbase/quotedblbase/perthousand
/Acircumflex/Ecircumflex/Aacute/Edieresis/Egrave/Iacute/Icircumflex/Idieresis/Igrave
/Oacute/Ocircumflex/apple/Ograve/Uacute/Ucircumflex/Ugrave/dotlessi/circumflex/tilde
/macron/breve/dotaccent/ring/cedilla/hungarumlaut/ogonek/caron
MacEncoding 128 128 getinterval astore pop
level2 startnoload
/copyfontdict
{
findfont dup length dict
begin
{
1 index/FID ne{def}{pop pop}ifelse
}forall
}bd
level2 endnoload level2 not startnoload
/copyfontdict
{
findfont dup length dict
copy
begin
}bd
level2 not endnoload
md/fontname known not{
/fontname/customfont def
}if
/Encoding Z
/:mre
{
copyfontdict
/Encoding MacEncoding def
fontname currentdict
end
definefont :ff def
}bd
/:bsr
{
copyfontdict
/Encoding Encoding 256 array copy def
Encoding dup
}bd
/pd{put dup}bd
/:esr
{
pop pop
fontname currentdict
end
definefont :ff def
}bd
/scf
{
scalefont def
}bd
/scf-non
{
$m scale :mf setfont
}bd
/ps Z
/fz{/ps xs}bd
/sf/setfont ld
/cF/currentfont ld
/mbf
{
/makeblendedfont where
{
pop
makeblendedfont
/ABlend exch definefont
}{
pop
}ifelse
def
}def
%%EndFile
%%BeginFile: adobe_psp_derived_styles
%%Copyright: Copyright 1990-1993 Adobe Systems Incorporated. All Rights Reserved.
/wi
version(23.0)eq
{
{
gS 0 0 0 0 rC stringwidth gR
}bind
}{
/stringwidth load
}ifelse
def
/$o 1. def
/gl{$o G}bd
/ms{:M S}bd
/condensedmtx[.82 0 0 1 0 0]def
/:mc
{
condensedmtx :mf def
}bd
/extendedmtx[1.18 0 0 1 0 0]def
/:me
{
extendedmtx :mf def
}bd
/basefont Z
/basefonto Z
/dxa Z
/dxb Z
/dxc Z
/dxd Z
/dsdx2 Z
/bfproc Z
/:fbase
{
dup/FontType get 0 eq{
dup length dict begin
dup{1 index/FID ne 2 index/UniqueID ne and{def}{pop pop}ifelse}forall
/FDepVector exch/FDepVector get[exch/:fbase load forall]def
}/bfproc load ifelse
/customfont currentdict end definefont
}bd
/:mo
{
/bfproc{
dup dup length 2 add dict
begin
{
1 index/FID ne 2 index/UniqueID ne and{def}{pop pop}ifelse
}forall
/PaintType 2 def
/StrokeWidth .012 0 FontMatrix idtransform pop def
/customfont currentdict
end
definefont
8 dict begin
/basefonto xdf
/basefont xdf
/FontType 3 def
/FontMatrix[1 0 0 1 0 0]def
/FontBBox[0 0 1 1]def
/Encoding StandardEncoding def
/BuildChar
{
exch begin
basefont setfont
( )dup 0 4 -1 roll put
dup wi
setcharwidth
0 0 :M
gS
gl
dup show
gR
basefonto setfont
show
end
}def
}store :fbase
}bd
/:mso
{
/bfproc{
7 dict begin
/basefont xdf
/FontType 3 def
/FontMatrix[1 0 0 1 0 0]def
/FontBBox[0 0 1 1]def
/Encoding StandardEncoding def
/BuildChar
{
exch begin
sD begin
/dxa 1 ps div def
basefont setfont
( )dup 0 4 -1 roll put
dup wi
1 index 0 ne
{
exch dxa add exch
}if
setcharwidth
dup 0 0 ms
dup dxa 0 ms
dup dxa dxa ms
dup 0 dxa ms
gl
dxa 2. div dup ms
end
end
}def
}store :fbase
}bd
/:ms
{
/bfproc{
dup dup length 2 add dict
begin
{
1 index/FID ne 2 index/UniqueID ne and{def}{pop pop}ifelse
}forall
/PaintType 2 def
/StrokeWidth .012 0 FontMatrix idtransform pop def
/customfont currentdict
end
definefont
8 dict begin
/basefonto xdf
/basefont xdf
/FontType 3 def
/FontMatrix[1 0 0 1 0 0]def
/FontBBox[0 0 1 1]def
/Encoding StandardEncoding def
/BuildChar
{
exch begin
sD begin
/dxb .05 def
basefont setfont
( )dup 0 4 -1 roll put
dup wi
exch dup 0 ne
{
dxb add
}if
exch setcharwidth
dup dxb .01 add 0 ms
0 dxb :T
gS
gl
dup 0 0 ms
gR
basefonto setfont
0 0 ms
end
end
}def
}store :fbase
}bd
/:mss
{
/bfproc{
7 dict begin
/basefont xdf
/FontType 3 def
/FontMatrix[1 0 0 1 0 0]def
/FontBBox[0 0 1 1]def
/Encoding StandardEncoding def
/BuildChar
{
exch begin
sD begin
/dxc 1 ps div def
/dsdx2 .05 dxc 2 div add def
basefont setfont
( )dup 0 4 -1 roll put
dup wi
exch dup 0 ne
{
dsdx2 add
}if
exch setcharwidth
dup dsdx2 .01 add 0 ms
0 .05 dxc 2 div sub :T
dup 0 0 ms
dup dxc 0 ms
dup dxc dxc ms
dup 0 dxc ms
gl
dxc 2 div dup ms
end
end
}def
}store :fbase
}bd
/:msb
{
/bfproc{
7 dict begin
/basefont xdf
/FontType 3 def
/FontMatrix[1 0 0 1 0 0]def
/FontBBox[0 0 1 1]def
/Encoding StandardEncoding def
/BuildChar
{
exch begin
sD begin
/dxd .03 def
basefont setfont
( )dup 0 4 -1 roll put
dup wi
1 index 0 ne
{
exch dxd add exch
}if
setcharwidth
dup 0 0 ms
dup dxd 0 ms
dup dxd dxd ms
0 dxd ms
end
end
}def
}store :fbase
}bd
/italicmtx[1 0 -.212557 1 0 0]def
/:mi
{
italicmtx :mf def
}bd
/:v
{
[exch dup/FontMatrix get exch
dup/FontInfo known
{
/FontInfo get
dup/UnderlinePosition known
{
dup/UnderlinePosition get
2 index 0
3 1 roll
transform
exch pop
}{
.1
}ifelse
3 1 roll
dup/UnderlineThickness known
{
/UnderlineThickness get
exch 0 3 1 roll
transform
exch pop
abs
}{
pop pop .067
}ifelse
}{
pop pop .1 .067
}ifelse
]
}bd
/$t Z
/$p Z
/$s Z
/:p
{
aload pop
2 index mul/$t xs
1 index mul/$p xs
.012 mul/$s xs
}bd
/:m
{gS
0 $p rm
$t lw
0 rl stroke
gR
}bd
/:n
{
gS
0 $p rm
$t lw
0 rl
gS
gl
stroke
gR
strokepath
$s lw
/setstrokeadjust where{pop
currentstrokeadjust true setstrokeadjust stroke setstrokeadjust
}{
stroke
}ifelse
gR
}bd
/:o
{gS
0 $p rm
$t 2 div dup rm
$t lw
dup 0 rl
stroke
gR
:n
}bd
%%EndFile
/currentpacking where {pop sc_oldpacking setpacking}if
end % md
%%EndProlog
%%BeginSetup
md begin
countdictstack
[
{%stopped
%%BeginFeature: *ManualFeed False
1 dict dup /ManualFeed false put setpagedevice
%%EndFeature
}featurecleanup
countdictstack
[
{%stopped
%%BeginFeature: *InputSlot Cassette
%%EndFeature
}featurecleanup
countdictstack
[
{%stopped
%%BeginFeature: *PageRegion LetterSmall
<> >>
/ImagingBBox [25 25 587 767] >> setpagedevice
%%EndFeature
}featurecleanup
(steve)setjob
/pT[1 0 0 -1 31 761]def/mT[1 0 0 -1 31 761]def
/sD 16 dict def
300 level2{1 dict dup/WaitTimeout 4 -1 roll put setuserparams}{statusdict/waittimeout 3 -1 roll put}ifelse
%%IncludeFont: Times-Roman
%%IncludeFont: Times-Bold
%%IncludeFont: Times-Italic
%%IncludeFont: Palatino-Roman
%%IncludeFont: Helvetica
/f0_1/Times-Roman :mre
/f0_18 f0_1 18 scf
/f0_14 f0_1 14 scf
/f0_12 f0_1 12 scf
/f0_10 f0_1 10 scf
/f0_9 f0_1 9 scf
/f1_1/Times-Bold :mre
/f1_18 f1_1 18 scf
/f1_14 f1_1 14 scf
/f1_12 f1_1 12 scf
/f1_9 f1_1 9 scf
/f2_1/Times-Italic :mre
/f2_12 f2_1 12 scf
/f2_9 f2_1 9 scf
/f3_1 f2_1 :v def
/f4_1 f0_1 :v def
/f5_1/Palatino-Roman :mre
/f5_18 f5_1 18 scf
/f5_12 f5_1 12 scf
/f5_9 f5_1 9 scf
/f6_1/Helvetica :mre
/f6_18 f6_1 18 scf
/Courier findfont[10 0 0 -10 0 0]:mf setfont
%%EndSetup
%%Page: 1 1
%%BeginPageSetup
initializepage
(steve; page: 1 of 29)setjob
%%EndPageSetup
gS 0 0 552 730 rC
5 13 :M
f0_9 sf
( )S
266 13 :M
( )S
522 13 :M
f1_9 sf
(1)S
187 43 :M
f1_18 sf
-.02(Integrative evaluation:)A
94 67 :M
-.008(An emerging role for classroom studies of CAL)A
79 92 :M
f2_12 sf
-.027(This paper appeared in )A
12 f3_1 :p
125.305 :m
-.031(Computers and Education)A
320 92 :M
-.021( \(1996\) vol.26 no.1-3, pp.17-32)A
87 140 :M
f0_12 sf
-.011( Stephen W. Draper, Margaret I. Brown, Fiona P. Henderson, Erica McAteer)A
459 137 :M
f5_9 sf
(*)S
210 164 :M
f0_12 sf
-.027(Department of Psychology)A
220 188 :M
-.015(University of Glasgow)A
212 212 :M
-.046(Glasgow G12 8QQ U.K.)A
209 236 :M
-.027(email: steve@psy.gla.ac.uk)A
174 260 :M
-.012(WWW: http://www.psy.gla.ac.uk/~steve)A
249 356 :M
f1_14 sf
-.011(Abstract)A
5 380 :M
f0_12 sf
-.008(This paper reviews the work of a team over two and a half years whose remit has been to "evaluate" a diverse)A
5 404 :M
-.009(range of CAL \(Computer Assisted Learning\) in use in a university setting. It gives an overview of the team's)A
5 428 :M
-.008(current method, including some of the instruments most often used, and describes some of the painful lessons)A
5 452 :M
-.004(from early attempts. It then offers a critical discussion of what the essential features of the method are, and of)A
5 476 :M
-.004(what such studies are and are not good for. One of the main conclusions, with hindsight, is that its main benefit)A
5 500 :M
-.009(is as "integrative evaluation": to help teachers make better use of the CAL by adjusting how it is used, rather)A
5 524 :M
-.014(than by changing the software or informing purchasing decisions.)A
228 596 :M
f1_14 sf
-.047(1. Introduction)A
5 620 :M
f0_12 sf
-.004(The authors constitute the evaluation group within a large project on CAL \(Computer Assisted Learning\) across)A
5 644 :M
-.001(a university, which has run for about two and half years at the time of writing [1]. We were charged with)A
-4127 -4126 -1 1 -4125 -4126 1 -4127 -4127 @a
5 657.24 -.24 .24 148.24 657 .24 5 657 @a
5 671 :M
f5_9 sf
(*)S
9 674 :M
f5_12 sf
( )S
f0_12 sf
-.001(This paper describes work by the evaluation group within the TILT project \(part of TLTP: see the)A
5 686 :M
-.004(acknowledgements\). Enquiries about this paper \(and other evaluation work\) should be sent to the first author at)A
5 698 :M
-.002(the address above. Enquiries about TILT generally should be sent to the project director:)A
5 710 :M
-.005(g.doughty@elec.gla.ac.uk or G.F.Doughty, Robert Clark Centre, 66 Oakfield Avenue, Glasgow G12 8LS, U.K.)A
endp
%%Page: 2 2
%%BeginPageSetup
initializepage
(steve; page: 2 of 29)setjob
%%EndPageSetup
-31 -31 :T
gS 31 31 552 730 rC
36 44 :M
f0_9 sf
( )S
297 44 :M
( )S
553 44 :M
f1_9 sf
(2)S
36 69 :M
f0_12 sf
-.001(evaluating the CAL courseware whose use was being developed by other members of the project. In response)A
36 93 :M
-.001(we have performed over 20 studies of teaching software in our university across a very wide range of subject)A
36 117 :M
-.008(disciplines, from Dentistry to Economic History, from Music to Engineering, from Zoology practicals to library)A
36 141 :M
-.007(skills. More detailed accounts of some of these studies and of some of our methods are available elsewhere)A
36 165 :M
-.004([2,3,4,5,6]. In this paper we review our experience as a whole, describe our current method, and discuss how it)A
36 189 :M
-.003(might be justified and what in fact it has turned out to be useful for.)A
197 261 :M
f1_14 sf
-.009(2. General features of our approach)A
36 285 :M
f0_12 sf
-.008(A popular term for the activity described here, and the one our project used for it, is "evaluation" \321 a term)A
36 309 :M
-.007(which implies making a \(value\) judgement. A better statement of our aim however is "to discover how an)A
36 333 :M
-.003(educational intervention performs" by observing and measuring the teaching and learning process, or some)A
36 357 :M
-.007(small slice of it. Our function is to provide better information than is ordinarily available about what is going)A
36 381 :M
-.001(on and its effects; it is up to others, most likely the teachers concerned, to use that information. One use would)A
36 405 :M
-.009(be to make judgements e.g. about whether to continue to use some piece of software \("summative" evaluation\).)A
36 429 :M
-.006(Other uses might be to improve the design of the software \("formative" evaluation\), to change how it is used)A
36 453 :M
-.005(\(e.g. support it by handouts and tutorials \321 "integrative" evaluation\), or to demonstrate the learning quality and)A
36 477 :M
-.018(quantity achieved \(a QA function\).)A
36 525 :M
-.001(Our basic aim, then, was to observe and measure what we could about the courseware and its effects. Practical)A
36 549 :M
-.005(constraints would place some limits on what we could do towards this aim. Over time we learned a lot about)A
36 573 :M
-.002(how and how not to run such studies. Now we are in a position to review what uses studies under these)A
36 597 :M
-.012(conditions turn out to have: not exactly the uses we originally envisaged.)A
36 645 :M
-.005(Our starting point was influenced by standard summative studies. We felt we should be able to answer)A
36 669 :M
-.004(questions about the effect of the courseware, and that this meant taking pre- and post-intervention measures.)A
36 693 :M
-.009(Furthermore, we felt these measures should derive from the students \(not from on-lookers' opinions\). This led)A
36 717 :M
-.005(to studies of the actual classes the software is designed for, and the need for instruments that can be)A
endp
%%Page: 3 3
%%BeginPageSetup
initializepage
(steve; page: 3 of 29)setjob
%%EndPageSetup
-31 -31 :T
gS 31 31 552 730 rC
36 44 :M
f0_9 sf
( )S
297 44 :M
( )S
553 44 :M
f1_9 sf
(3)S
36 69 :M
f0_12 sf
-.007(administered to all students before and after the courseware. Such studies have the great virtue of being in a)A
36 93 :M
-.01(strong position to achieve validity: having realistic test subjects in real conditions.)A
36 141 :M
-.001(Various pressures contribute to maintaining an emphasis on these special classroom opportunities, despite some)A
36 165 :M
-.033(drawbacks.)A
36 189 :M
(1.)S
63 189 :M
-.001(We are most likely to be invited to do a study \(or, if we take the initiative, to secure agreement\) when new)A
63 213 :M
-.002(courseware is being introduced to classes. There are several reasons for this.)A
36 237 :M
(1.1)S
63 237 :M
-.009(This corresponds with many people's idea of evaluation as a one-shot, end of project, summative activity.)A
63 261 :M
-.004(Thus it is usually the default idea of developers, authors, teachers, funding bodies, universities etc.. The)A
63 285 :M
-.001(desirability of this is discussed below, but meanwhile it has a large effect on expectations, and hence on)A
63 309 :M
-.025(opportunities.)A
36 333 :M
(1.2)S
63 333 :M
-.001(Experimental technique requires pre- and post-tests using quantitative measures in controlled conditions.)A
63 357 :M
-.008(This means that a lot of effort from the students, the teaching staff, and the investigators is put into each)A
63 381 :M
-.005(such occasion; and so it is unlikely that they can afford to do this often. Once a year is all that can be)A
63 405 :M
-.041(easily afforded.)A
36 429 :M
(2.)S
63 429 :M
-.004(The most important criterion in testing is whether the courseware is effective i.e. do students learn from it.)A
63 453 :M
-.005(It is hard to test this without a complete implementation.)A
36 477 :M
(3.)S
63 477 :M
-.003(It is important to get test subjects who are motivated to learn, and don't know the material already. These)A
63 501 :M
-.003(are often only available once a year in the target classes.)A
36 525 :M
(4.)S
63 525 :M
-.008(An advantage of this that we came to appreciate is that then the whole situation is tested, not just the)A
63 549 :M
-.002(courseware which in reality is only one factor in determining learning outcomes.)A
36 573 :M
(5.)S
63 573 :M
-.005(One important limit is the need not to overload the students. As investigators, we were happy to keep on)A
63 597 :M
-.008(multiplying the tests, questionnaires and interviews in order to learn more, but we quickly learned that)A
63 621 :M
-.008(students have a strictly limited tolerance for this addition to their time and trouble on such occasions.)A
63 645 :M
-.005(Hence potential instruments must compete with each other at minimising their cost to the students.)A
36 669 :M
(6.)S
63 669 :M
-.004(Investigator time is also a limiting factor. There are always far fewer investigators than students, so at)A
63 693 :M
-.008(best only a small sample can be interviewed and individually observed. We must therefore concentrate on)A
63 717 :M
-.037(paper instruments.)A
endp
%%Page: 4 4
%%BeginPageSetup
initializepage
(steve; page: 4 of 29)setjob
%%EndPageSetup
-31 -31 :T
gS 31 31 552 730 rC
36 44 :M
f0_9 sf
( )S
297 44 :M
( )S
553 44 :M
f1_9 sf
(4)S
36 69 :M
f0_12 sf
-.005(Thus our method focusses around such big occasions, and in effect is organised to make the most of these)A
36 93 :M
-.008(opportunities: to observe and measure what can be observed under these conditions. We tend to study real)A
36 117 :M
-.004(students learning as part of their course. We rely on paper instruments \(e.g. questionnaires, multiple choice)A
36 141 :M
-.008(tests\) to survey the whole class, supplemented by personal observations and interviews with a sample.)A
216 213 :M
f1_14 sf
-.03(3. Overview of our "method")A
36 237 :M
f0_12 sf
-.004(Each particular study is designed separately, depending upon the goals of the evaluation, upon the particular)A
36 261 :M
-.008(courseware to be studied, and upon the teaching situation in which it is to be used. Each design draws on our)A
36 285 :M
-.001(battery of methods and instruments, which are still actively evolving, and which are selected and adapted anew)A
36 309 :M
-.005(for each study. In this section we describe what is common in our approach across most studies, but the)A
36 333 :M
-.006(considerable degree of individual adaptation means that the reader should understand that the term "method")A
36 357 :M
-.004(should be read as having inverted commas: it has not been a fixed procedure mechanically applied. On the)A
36 381 :M
-.007(other hand, there has been a substantial uniformity in what we did despite great variations in the material being)A
36 405 :M
-.001(taught and in performing both formative evaluations of unfinished software and studies of finished software in)A
36 429 :M
-.042(its final classroom use.)A
36 477 :M
(Such studies are a collaborative effort between evaluators \(ourselves\), teachers \(i.e. the lecturer or professor)S
36 501 :M
-.004(responsible for running the teaching activity plus his or her assistants\), students, and \(if the software is produced)A
36 525 :M
-.007(in-house\) developers \(the designers and writers of the software\). Any failure to secure willing cooperation from)A
36 549 :M
-.008(the students was marked by a drop in the the number and still more in the usefulness of the returns. The)A
36 573 :M
-.006(teacher's cooperation is still more central: not only for permission and class time, but for the design of tests of)A
36 597 :M
-.004(learning gains, and interpretation of the results to which, as the domain experts, they are crucial contributors.)A
36 621 :M
-.007(As a rule, the evaluators will have most to say about the basic data, both evaluators and teachers will contribute)A
36 645 :M
-.007(substantially to interpretations of the probable issues and causes underlying the observations, and the teachers or)A
36 669 :M
-.002(developers will have most to say about what action if any they may now plan as a consequence.)A
36 717 :M
-.002(Our method can be seen as having two levels. The "outer method" concerns collaborating with the teachers and)A
36 741 :M
-.004(developers to produce and agree a design for the study, carry it out at an identified opportunity, and produce a)A
endp
%%Page: 5 5
%%BeginPageSetup
initializepage
(steve; page: 5 of 29)setjob
%%EndPageSetup
-31 -31 :T
gS 31 31 552 730 rC
36 44 :M
f0_9 sf
( )S
297 44 :M
( )S
553 44 :M
f1_9 sf
(5)S
36 69 :M
f0_12 sf
-.007(report including input from the teacher about what the observations might mean. Within that is an "inner)A
36 93 :M
-.009(method" for the selection and design of specific instruments and observational methods.)A
36 141 :M
f1_12 sf
-.014(3.1 Our "outer method": interaction with teachers and developers)A
36 165 :M
f0_12 sf
-.01(Generally we would follow a pattern summarised as follows:)A
36 213 :M
(1.)S
63 213 :M
-.004(One or more meetings, perhaps prepared for by a previous elicitation of relevant information by mail, of)A
63 237 :M
-.014(evaluators with teachers or developers to:)A
36 261 :M
(\245)S
63 261 :M
-.001(Establish the teachers' goals for the evaluation)A
36 285 :M
(\245)S
63 285 :M
-.018(Elicit the learning aims and objectives to be studied)A
36 309 :M
(\245)S
63 309 :M
-.008(Elicit the classroom provision, course integration, and other features of the teaching situation and support)A
63 333 :M
-.001(surrounding the courseware itself e.g. how the material is assessed, whether it is a scheduled teaching)A
63 357 :M
-.047(event or open access.)A
36 381 :M
(\245)S
63 381 :M
-.01(Establish options for classroom observation, questionnaire administration, interviews, etc.)A
36 405 :M
(\245)S
63 405 :M
-.002(Discuss the feasibility of creating a learning quiz or other measure of learning gains.)A
36 453 :M
(2.)S
63 453 :M
-.006(An evaluator may go through the software to get an impression of what is involved there.)A
36 501 :M
(3.)S
63 501 :M
-.004(The teacher creates \(if feasible\) a quiz with answers and marking scheme, and defines any other)A
63 525 :M
-.054(assessment methods.)A
36 573 :M
(4.)S
63 573 :M
-.004(Evaluator and teacher finalise a design for the study.)A
36 621 :M
(5.)S
63 621 :M
-.029(Classroom study occurs.)A
36 669 :M
(6.)S
63 669 :M
-.007(A preliminary report from the evaluators is presented to the teacher, and interpretations of the findings)A
63 693 :M
-.016(sought and discussed.)A
36 741 :M
(7.)S
63 741 :M
-.046(Final report produced.)A
endp
%%Page: 6 6
%%BeginPageSetup
initializepage
(steve; page: 6 of 29)setjob
%%EndPageSetup
-31 -31 :T
gS 31 31 552 730 rC
36 44 :M
f0_9 sf
( )S
297 44 :M
( )S
553 44 :M
f1_9 sf
(6)S
36 69 :M
f1_12 sf
-.026(3.2 "Inner method": some instruments)A
36 93 :M
f0_12 sf
-.005(Every study is different, but a prototypical design of a large study might use all of the following instruments.)A
36 141 :M
12 f4_1 :p
172.001 :m
-.018(Computer Experience questionnaire)A
36 165 :M
-.001(This asks about past and current computer training, usage, skills, attitudes to and confidence about computers)A
36 189 :M
-.007(and computer assisted learning. We use this less and less, as has turned out to be seldom an important factor.)A
36 237 :M
150.001 :m
-.021(Task Experience Questionnaire)A
36 261 :M
-0(Where particular skills are targeted by courseware, it can be useful for teachers to have some information about)A
36 285 :M
-.006(students' current experience in the domain. If possible this questionnaire should be administered to students at a)A
36 309 :M
-.004(convenient point which is prior to the class within which the courseware evaluation itself is to run. \(This is not)A
36 333 :M
-.005(to be confused with diagnostic tests to establish individual learning needs, which might be provided so that)A
36 357 :M
-.007(students can enter a series of teaching units at an appropriate level\).)A
36 405 :M
237.999 :m
-.013(Observations \(by evaluator, possibly using video\))A
36 429 :M
-.001(Whenever possible we have at least one evaluator at a site as an observer, gathering open-ended observations.)A
36 453 :M
-.001(Sometimes we have set up a video camera to observe one computer station, while the human observers might)A
36 477 :M
-.048(watch another.)A
36 525 :M
114.867 :m
-.019(Student confidence logs)A
-.02( \(Example in appendix A.\))A
36 549 :M
-.003(These are checklists of specific learning objectives for a courseware package, provided by the teacher, on which)A
36 573 :M
-.004(students rate their confidence about their grasp of underlying principles or their ability to perform tasks.)A
36 597 :M
-.001(Typically they do this immediately before encountering a set of learning materials or engaging in a learning)A
36 621 :M
-.007(exercise, then again immediately afterward. If there are several exposures to the information in different)A
36 645 :M
-.008(settings \321 tutorials, practical labs, lectures, independent study opportunities, say \321 then they may be asked to)A
36 669 :M
-.004(complete them at several times. A rise in scores for an item upon exposure to a particular activity shows that)A
36 693 :M
-.004(the students at least believe they have learned from it, while no rise makes it unlikely to have been of benefit.)A
36 717 :M
-.007(Since this instrument measures confidence rather than a behavioural demonstration of knowledge like an exam,)A
endp
%%Page: 7 7
%%BeginPageSetup
initializepage
(steve; page: 7 of 29)setjob
%%EndPageSetup
-31 -31 :T
gS 31 31 552 730 rC
36 44 :M
f0_9 sf
( )S
297 44 :M
( )S
553 44 :M
f1_9 sf
(7)S
36 69 :M
f0_12 sf
-.001(it is only an indirect indication to be interpreted with some caution. Nevertheless, simple to use, these logs are)A
36 93 :M
-.004(proving to be an unexpectedly useful diagnostic instrument.)A
36 141 :M
12 f4_1 :p
93.765 :m
-.031(Knowledge quizzes)A
-.029( \(Example in appendix B.\))A
36 165 :M
-.001(These are constructed by the teacher and given to students before and after an intervention, and at a later point)A
36 189 :M
-.001(\(delayed post-test\) where sensible. Each question usually corresponds to a distinct learning objective. For)A
36 213 :M
-.007(consistent marking, these quizzes are usually multiple choice. Their purpose is assessment of the courseware)A
36 237 :M
-.001(and other teaching \321 low average post-test scores on a particular question point to a learning objective that)A
36 261 :M
-.001(needs better, or different, treatment. High average pre-test scores may suggest that certain content is redundant.)A
36 285 :M
-.001(They can sometimes be usefully incorporated as a permanent part of the program, focusing attention before)A
36 309 :M
-.013(study, or for self assessment after a session.)A
36 357 :M
150 :m
-.034(Post Task Questionnaire \(PTQ\))A
187 357 :M
-.038( \(Example in appendix C.\))A
36 381 :M
-.002(This is usually given immediately after the student has completed the class session which involves use of the)A
36 405 :M
-.007(courseware. It can be extended to fit other, related, classwork with optional questions depending on its occasion)A
36 429 :M
-.005(of use. It gathers "survey" information, at a fairly shallow level, about what students felt they were doing)A
36 453 :M
-.004(during the learning task, and why. It can also ask specific questions about points in which teachers, developers)A
36 477 :M
-.001(or evaluators are specifically interested \321 e.g. the use and usefulness of a glossary, perceived relevance of)A
36 501 :M
-.008(content or activity to the course in which it is embedded, etc. Also some evaluative judgements by students can)A
36 525 :M
-.001(be sought if wanted at that point. Ideally the information gained \(which we are still seeking to extend as we)A
36 549 :M
-.005(develop the instrument further\) should be supplemented by sample interviews and/or focus groups. The)A
36 573 :M
-.001(Resource Questionnaire \(see below\) shares some content items with the PTQ - where appropriate, questions)A
36 597 :M
-.003(could be loaded on to that, to prevent too much use of student time during class.)A
36 645 :M
255 :m
-.005(Focus groups or interviews with a sample of students)A
36 669 :M
-.002(Focus groups consist of a group of students and an investigator, who will have a few set questions to initiate)A
36 693 :M
-.001(discussion. The students' comments act as prompts for each other, often more important than the investigator's)A
36 717 :M
-.005(starting questions. Both focus groups and interviews have two purposes. Firstly, as a spoken method of)A
36 741 :M
-.008(administering fixed questions \(e.g. from the PTQ\) they allow a check on the quality of the written responses)A
endp
%%Page: 8 8
%%BeginPageSetup
initializepage
(steve; page: 8 of 29)setjob
%%EndPageSetup
-31 -31 :T
gS 31 31 552 730 rC
36 44 :M
f0_9 sf
( )S
297 44 :M
( )S
553 44 :M
f1_9 sf
(8)S
36 69 :M
f0_12 sf
-.001(from the rest of the students. Secondly they are used as an open-ended instrument to elicit points that were not)A
36 93 :M
-.001(foreseen when designing the questionnaires. Generally we get far more detail from these oral accounts than)A
36 117 :M
-.001(from written ones, especially if the answers are probed by follow up questions, and in addition we can ask for)A
36 141 :M
-.018(clarification on the spot of any unclear statements.)A
36 189 :M
12 f4_1 :p
161 :m
-.009(Learning Resource Questionnaire)A
36 213 :M
-.004(With the teachers, we produce a checklist of available learning resources \(e.g. lectures, courseware, books,)A
36 237 :M
-.001(tutorials, other students etc.\) and ask students to indicate, against each item, whether they used it, how useful it)A
36 261 :M
-.001(was, how easily accessed, how critical for their study etc. We would normally administer this during a course,)A
36 285 :M
-.006(during exam revision, and perhaps after an exam. It has two main functions. The first is to look at students')A
36 309 :M
-.005(independent learning strategies in the context of a department's teaching resources \(which do they seek out and)A
36 333 :M
-.002(spend time on\). The second is to evaluate those resources \(which do they actually find useful\). This)A
36 357 :M
-.007(questionnaire is especially useful for courses within which the computer package is available in open access)A
36 381 :M
-.008(labs, when it is not possible to monitor classes at a sitting by Post Task Questionnaires. This instrument is)A
36 405 :M
-.011(discussed in detail in [4].)A
36 453 :M
125.001 :m
-.013(Post Course questionnaire)A
36 477 :M
-.001(Sometimes there are useful questions which can be asked of students at the end of a course or section of course)A
36 501 :M
-.008(which has involved the use of CAL. Teachers may seek information to expand or clarify unexpected)A
36 525 :M
-.001(observations made while the classes were in progress. We might want to get an overview from the students, to)A
36 549 :M
-.003(put with their daily "post task" feedback, or we may want to ask questions that may have seemed intrusive at the)A
36 573 :M
-.004(start of the course but, once students are familiar with course content, the evaluation itself and the evaluators,)A
36 597 :M
-.007(are more naturally answered. This instrument can be useful where a resource questionnaire \(a more usual place)A
36 621 :M
-.012(for "post course" general questions\) is not appropriate.)A
36 669 :M
360.998 :m
-.003(Access to subsequent exam performance on one or more relevant questions)A
36 693 :M
-.001(In some cases it is later possible to obtain access to relevant exam results. Exam questions are not set to please)A
36 717 :M
-.007(evaluators, so only occasionally is there a clear correspondence between an exam item and a piece of CAL.)A
endp
%%Page: 9 9
%%BeginPageSetup
initializepage
(steve; page: 9 of 29)setjob
%%EndPageSetup
-31 -31 :T
gS 31 31 552 730 rC
36 44 :M
f0_9 sf
( )S
297 44 :M
( )S
553 44 :M
f1_9 sf
(9)S
224 72 :M
f1_14 sf
-.026(4. Some problematic issues)A
36 96 :M
f0_12 sf
-.004(In the course of our studies, a number of issues struck us with some force. Often they were epitomised by a)A
36 120 :M
-.014(specific experience.)A
36 168 :M
f1_12 sf
-.05(General points)A
36 192 :M
f0_12 sf
12 f4_1 :p
189.999 :m
-.007(Subjects with the right prior knowledge)A
36 216 :M
-.008(One of us was called on to comment on a piece of CAL being developed on the biochemistry of lipid synthesis.)A
36 240 :M
-.004(As he had no background knowledge of biochemistry at all, he was inclined to suggest that the courseware was)A
36 264 :M
-.001(too technical and offered no handle to the backward students who might need it most. In fact, tests on the target)A
36 288 :M
-.007(set of students showed that this was quite wrong, and that they all were quite adequately familiar with the basic)A
36 312 :M
-.005(notation required by the courseware. This shows the importance, however informal the testing, of finding)A
36 336 :M
-.001(subjects with sufficient relevant pre-requisite knowledge for the material. It is not possible to hire subjects from)A
36 360 :M
-.005(the general population \(not even the general undergraduate population\) to test much of the CAL in higher)A
36 384 :M
-.072(education.)A
36 432 :M
-.002(The converse point, perhaps more familiar but equally important, is that using subjects with too much)A
36 456 :M
-.005(knowledge is equally likely to be misleading. Hence asking teachers to comment is of limited value as they)A
36 480 :M
-.004(cannot easily imagine what it is like not to know the material already. Similarly, in one of our formative studies)A
36 504 :M
-.004(of a statistical package, we used students from a later point in the course who had already covered the material.)A
36 528 :M
-.004(On the few points where they had difficulty we could confidently conclude there was a problem to be cleared)A
36 552 :M
-.001(up, but when they commented that on the whole they felt the material was too elementary we could not tell)A
36 576 :M
-.001(whether to accept that as a conclusion or that it was simply because they already knew the material. This)A
36 600 :M
-.001(illustrates how, while some conclusions can be drawn from other subjects \(if over-qualified subjects have)A
36 624 :M
-.001(difficulty or under-qualified subjects find it trivial then there must be a problem\), only subjects with exactly the)A
36 648 :M
-.001(right prior knowledge qualifications are most useful. This contributes to the pressure on using real target)A
36 672 :M
-.027(classes, despite the limited supply.)A
endp
%%Page: 10 10
%%BeginPageSetup
initializepage
(steve; page: 10 of 29)setjob
%%EndPageSetup
-31 -31 :T
gS 31 31 552 730 rC
36 44 :M
f0_9 sf
( )S
297 44 :M
( )S
549 44 :M
f1_9 sf
(10)S
36 69 :M
f0_12 sf
12 f4_1 :p
280.001 :m
-.012(The evaluators' dependence on collaboration with teachers)A
36 93 :M
-.001(The same experiences and considerations also show that evaluators are often wholly unqualified in domain)A
36 117 :M
-.004(knowledge especially for university courses, whether of biochemistry or of sixteenth century musicianship.)A
36 141 :M
-.005(Teachers are qualified in the domain knowledge, and so evaluators must depend on them for this \(e.g. in)A
36 165 :M
-.004(designing knowledge quizzes, and in judging the answers\), as well as for interpreting findings \(whether of poor)A
36 189 :M
-.004(scores or of student comments\) in terms of what the most likely causes and remedies are. On the other hand)A
36 213 :M
-.009(teachers are over-qualified in knowledge, hence the value of doing studies with real students.)A
36 261 :M
201.999 :m
-.015(Subjects with the right motivation to learn)A
36 285 :M
-.001(Subjects must equally have the right motivation to learn \(i.e. the same as in the target situation\). Where would)A
36 309 :M
-.001(you find subjects naturally interested in learning about statistics or lipid synthesis? Such people are rare, and in)A
36 333 :M
-.004(any case not representative of the target group, who in many cases only learn because it is a requirement of the)A
36 357 :M
-.005(course they are on. This was brought home to us in an early attempt to study a Zoology simulation package.)A
36 381 :M
-.007(The teacher announced its availability and recommended it to a class. The evaluators turned up, but not a single)A
36 405 :M
-.017(student. Only when it was made compulsory was it used.)A
36 453 :M
-.002(Paying subjects is not in general a solution, although it may be worth it for early trials, where some useful)A
36 477 :M
-.005(results can be obtained without exactly reproducing the target conditions. Sometimes people are more)A
36 501 :M
-.001(motivated by helping in an experiment than they are by other factors. For instance if you say you are doing)A
36 525 :M
-.004(research on aerobic exercise you may easily persuade people to run round the block, but if you ask your friends)A
36 549 :M
-.005(to run round the block because watching them sweat gives you pleasure you are more likely to get a punch on)A
36 573 :M
-.001(the nose than compliance. In other words, participation in research may produce more motivation than)A
36 597 :M
-.008(friendship can. On the other hand with educational materials, paid subjects may well process the material)A
36 621 :M
-.005(politely but without the same kind of attempt to engage with it that wanting to earn a qualification may produce.)A
36 645 :M
-.008(In general, as the literature on cognitive dissonance showed, money probably produces a different kind of)A
36 669 :M
-.004(motivation. The need for the right motivation in subjects, like the need for the right level of knowledge, is a)A
36 693 :M
-.008(pressure towards using the actual target classes in evaluation studies.)A
endp
%%Page: 11 11
%%BeginPageSetup
initializepage
(steve; page: 11 of 29)setjob
%%EndPageSetup
-31 -31 :T
gS 31 31 552 730 rC
36 44 :M
f0_9 sf
( )S
297 44 :M
( )S
549 44 :M
f1_9 sf
(11)S
36 69 :M
f1_12 sf
-.044(Measures)A
36 93 :M
f0_12 sf
12 f4_1 :p
86 :m
-.06(Attitude measures)A
36 117 :M
-.004(Asking students at the end of an educational intervention \(EI \321 for example a piece of courseware\) what they)A
36 141 :M
-.004(feel about it is of some interest, as teachers would prefer that students enjoy courses. However learning gains)A
36 165 :M
-.009(are a far more important outcome in most teachers' view, and attitudes are very weak as a measure of that)A
36 189 :M
-.007(educational effect. Attitudes seem to be mainly an expression of how the experience compared to expectations.)A
36 213 :M
-.004(With CAL, at least currently, attitudes vary widely even within a class but above all across classes in different)A
36 237 :M
-.001(parts of the university. For instance we have observed some student groups who had the feeling that CAL was)A
36 261 :M
-.007(state of the art material that they were privileged to experience, and other groups who viewed it as a device by)A
36 285 :M
-.004(negligent staff to avoid the teaching which the students had paid for. In one case, even late in a course of)A
36 309 :M
-.008(computer mediated seminars that in the judgement of the teaching staff and on measures of content and)A
36 333 :M
-.004(contributions was much more successful than the non-computer form they replaced, about 10% of the students)A
36 357 :M
-.001(were still expressing the view that if they had known in advance that they would be forced to do this they would)A
36 381 :M
-.001(have chosen another course. Thus students express quite strongly positive and negative views about a piece of)A
36 405 :M
-.007(courseware that often seem unrelated to their actual educational value. This view of attitudes being determined)A
36 429 :M
-.004(by expectation not by real value is supported by the corollary observation that when the same measures are)A
36 453 :M
-.007(applied to lectures \(as we did in a study that directly compared a lecture with a piece of CAL\), students did not)A
36 477 :M
-.004(express strong views. In followup focus groups, however, it emerged that they had low expectations of lectures,)A
36 501 :M
-.004(and their experience of them was as expected; whereas the CAL often elicited vocal criticisms even though)A
36 525 :M
-.008(they learned successfully from it, and overall brought out many more comments both positive and negative)A
36 549 :M
-.001(reflecting a wide range of attitudes, presumably because prior expectations were not uniform in this group.)A
36 597 :M
-.004(Measuring the shift in attitude instead \(i.e. replacing a single post-EI attitude measure by pre- and post-measures)A
36 621 :M
-.009(of attitude\) does not solve the problem. A big positive shift could just mean that the student had been)A
36 645 :M
-.001(apprehensive and was relieved by the actual experience, a big negative shift might mean they had had)A
36 669 :M
-.004(unrealistic expectations, and no shift would mean that they had had accurate expectations but might mean either)A
36 693 :M
-.021(great or small effectiveness.)A
endp
%%Page: 12 12
%%BeginPageSetup
initializepage
(steve; page: 12 of 29)setjob
%%EndPageSetup
-31 -31 :T
gS 31 31 552 730 rC
36 44 :M
f0_9 sf
( )S
297 44 :M
( )S
549 44 :M
f1_9 sf
(12)S
36 69 :M
f0_12 sf
-.004(These criticisms of attitude measures also apply in principle to the course feedback questionnaires now)A
36 93 :M
-.004(becoming widespread in UK universities. As has been shown \(e.g. [7]\), these have a useful level of validity)A
36 117 :M
-.005(because, we believe, students' expectations are reasonably well educated by their experience of other courses: so)A
36 141 :M
-.006(a course that is rated badly compared to others probably really does have problems. However students' widely)A
36 165 :M
-.004(varying expectations of CAL as opposed to more familiar teaching methods render these measures much less)A
36 189 :M
-.001(useful in this context, increasing the importance of other measures. Attitudes are still important to measure,)A
36 213 :M
-.002(however, as teachers are likely to want to respond to them, and perhaps attempt to manage them.)A
36 261 :M
12 f4_1 :p
78 :m
-.022(Confidence logs)A
36 285 :M
-.002(Confidence logs ask students, not whether they thought an EI was enjoyable or valuable, but whether they)A
36 309 :M
-.001(\(now\) feel confident of having attained some learning objective. Like attitude measures, they are an indirect)A
36 333 :M
-.004(measure of learning whose validity is open to question. They have been of great practical importance however)A
36 357 :M
-.001(because they take up much less student time than the quizzes that measure learning more directly, and so can be)A
36 381 :M
-.006(applied more frequently e.g once an hour during an afternoon of mixed activities.)A
36 429 :M
-.007(By and large confidence logs can be taken as necessary but not sufficient evidence: if students show no increase)A
36 453 :M
-.001(in confidence after an EI it is unlikely that they learned that item on that occasion, while if they do show an)A
36 477 :M
-.009(increase then corroboration from quiz scores is advisable as a check against over confidence on the students')A
36 501 :M
-.007(part. Even the occasional drop in confidence that we have measured is consistent with this view: in that case, a)A
36 525 :M
-.004(practical exercise seems to have convinced most students that they had more to learn to master the topic than)A
36 549 :M
-.004(they had realised. We have often used confidence logs several times during a long EI, and quizzes once at the)A
36 573 :M
-.001(end. In these conditions, the quizzes give direct evidence about whether each learning objective was achieved,)A
36 597 :M
-.006(while the confidence logs indicate which parts of the EI were important for this gain.)A
36 645 :M
94 :m
-.018(Knowledge quizzes)A
36 669 :M
-.007(Although knowledge quizzes are a relatively direct measure of learning, they too are subject to questions about)A
36 693 :M
-.004(their validity. When scores on one item are low this could be due either to poor delivery of material for that)A
36 717 :M
-.004(learning objective or because the quiz item was a poor test of it or otherwise defective. When we feed back the)A
36 741 :M
-.007(raw results to the teacher they may immediately reconsider the quiz item \(which they themselves designed\), or)A
endp
%%Page: 13 13
%%BeginPageSetup
initializepage
(steve; page: 13 of 29)setjob
%%EndPageSetup
-31 -31 :T
gS 31 31 552 730 rC
36 44 :M
f0_9 sf
( )S
297 44 :M
( )S
549 44 :M
f1_9 sf
(13)S
36 69 :M
f0_12 sf
-.005(reconsider the teaching. Although this may sound like a methodological swamp when described in the abstract,)A
36 93 :M
-.007(in practice such questions are usually resolved fairly easily by interviewing the students about the item. Thus)A
36 117 :M
-.004(our studies are not much different from other practical diagnostic tasks such as tracking down a domestic)A
36 141 :M
-.003(electrical fault: it is always possible that the "new" light bulb you put in was itself faulty, but provided you keep)A
36 165 :M
-.008(such possibilities in mind it is not hard to run cross checks that soon provide a consistent story of where the)A
36 189 :M
-.027(problem or problems lie.)A
36 237 :M
-.001(Much of the usefulness of both quizzes and confidence logs comes from their diagnostic power, which in turn)A
36 261 :M
-.004(comes from the way they are specific to learning objectives, each of which is tested as a separate item. For)A
36 285 :M
-.004(instance when one item shows little or no gain compared to the others, teachers can focus on improving delivery)A
36 309 :M
-.007(for that objective. Such differential results simultaneously give some confidence that the measures are working)A
36 333 :M
-.001(\(and not suffering from ceiling or floor effects\), that much of the learning is proceeding satisfactorily, that there)A
36 357 :M
-.007(are specific problems, and where those problems are located. In other words they both give confidence in the)A
36 381 :M
-.004(measures and strongly suggest \(to the teachers at least\) what might be done about the problems. In this respect)A
36 405 :M
-.008(they are much more useful than measures applied to interventions as a whole \(such as course feedback)A
36 429 :M
-.007(questionnaires\), which even when their results are accepted by teachers as indicating a problem \(which they)A
36 453 :M
-.015(often are not\), are not helpful in suggesting what to change.)A
36 501 :M
12 f4_1 :p
110 :m
-.029(Delayed learning gains)A
36 525 :M
-.001(In many educational studies, measures of learning outcomes are taken not only before and immediately after the)A
36 549 :M
-.002(EI, but also some weeks or months later at a delayed post-test. This is usually done to measure whether)A
36 573 :M
-.004(material, learned at the time, is soon forgotten. In higher education, however, there is an opposite reason: both)A
36 597 :M
-.007(staff and students often express the view that the important part of learning is not during EIs such as lectures or)A
36 621 :M
-.004(labs, but at revision time or other self-organised study times. If this is true, then there might be no gains shown)A
36 645 :M
-.012(at an immediate post-test, only at a delayed test.)A
36 693 :M
93 :m
-.04(Auto compensation)A
36 717 :M
-.01(This all seems to make it clear that the best measure would be the difference between pre-test and delayed post-)A
36 741 :M
-.001(test. However there is an inescapable problem with this: that any changes might be due to other factors, such as)A
endp
%%Page: 14 14
%%BeginPageSetup
initializepage
(steve; page: 14 of 29)setjob
%%EndPageSetup
-31 -31 :T
gS 31 31 552 730 rC
36 44 :M
f0_9 sf
( )S
297 44 :M
( )S
549 44 :M
f1_9 sf
(14)S
36 69 :M
f0_12 sf
-.007(revision classes, going to text books etc. Indeed this is very likely, both from external factors \(other activities)A
36 93 :M
-.005(relating to the same topic\), and also from a possible internal factor that might be called "auto compensation". In)A
36 117 :M
-.008(higher education students are given a lot of responsibility for their learning, and how they go about it. This)A
36 141 :M
-.001(means that if they find one resource unsatisfactory or unlikeable, they are likely to compensate by seeking out a)A
36 165 :M
-.001(remedial source. Thus, for instance, bad lectures may not cause bad exam results because the students will go)A
36 189 :M
-.007(to textbooks, and similarly great courseware may not cause better exam results but simply allow students to get)A
36 213 :M
-.014(the same effect with less work from books.)A
36 261 :M
-.007(Thus although we gather delayed test results \(e.g. from exams\) when we can, these really only give information)A
36 285 :M
-.004(on the effect of the course as a whole including all resources. Only immediate post-tests can pick out the effects)A
36 309 :M
-.007(of a specific EI such as a piece of CAL, even if important aspects of learning and reflection may only take place)A
36 333 :M
-.063(later.)A
36 381 :M
12 f4_1 :p
106 :m
-.015(Open-ended measures)A
36 405 :M
-.001(All the above points have been about measures that are taken systematically across the whole set of subjects and)A
36 429 :M
-.001(can be compared quantitatively. More important than all of them however are the less formal open-ended)A
36 453 :M
-.001(measures \(e.g. personal observations, focus groups, interviews with open-ended questions\). This is true firstly)A
36 477 :M
-.001(because most of what we have learned about our methods and how they should be improved has come from)A
36 501 :M
-.002(such unplanned observations: about what was actually going on as opposed to what we had planned to measure.)A
36 525 :M
-.005(Not only have we drawn on them to make the arguments here about big issues, but also they have often been)A
36 549 :M
-.002(important in interpreting small things. For instance in one formative study there was really only one serious)A
36 573 :M
-.002(problem where all the students got stuck, but not all of them commented on this clearly in the paper instruments.)A
36 597 :M
-.001(It was only our own observation that made this stand out clearly, and so made us focus on the paper measures)A
36 621 :M
-.007(around this point. Open-ended measures are also often valued by the teachers: transcribed comments are a rich)A
36 645 :M
-.012(source of insight to them.)A
36 693 :M
f1_12 sf
-.008(Other factors affecting learning)A
36 717 :M
f0_12 sf
-.007(Our studies were often thought of as studies of the software and its effects. The problem with this view was)A
36 741 :M
-.009(brought home to us in an early study of Zoology simulation software being used in a lab class as one)A
endp
%%Page: 15 15
%%BeginPageSetup
initializepage
(steve; page: 15 of 29)setjob
%%EndPageSetup
-31 -31 :T
gS 31 31 552 730 rC
36 44 :M
f0_9 sf
( )S
297 44 :M
( )S
549 44 :M
f1_9 sf
(15)S
36 69 :M
f0_12 sf
-.009("experiment" out of about six that students rotated between. We observed that the teacher running the class)A
36 93 :M
-.001(would wait until students at the software station had finished their basic run through and then engage them at)A
36 117 :M
-.002(this well chosen moment in a tutorial dialogue to bring out the important conceptual points. Obviously any)A
36 141 :M
-.008(learning gains recorded could not be simply ascribed to the software: they might very well depend on this)A
36 165 :M
-.002(human tutoring. This was not anything the documentation had suggested, nor anything the teacher had)A
36 189 :M
-.006(mentioned to us: it was basically unplanned good teaching. On the other hand, the teacher's first priority in)A
36 213 :M
-.004(being there was to manage the lab i.e. handling problems of people, equipment, materials would have had first)A
36 237 :M
-.006(priority, so there was no guarantee of these tutorial interactions being delivered.)A
36 285 :M
-.005(This showed that our studies could not be thought of as controlled experiments, but had the advantages and)A
36 309 :M
-.007(disadvantages of realistic classroom studies. We could have suppressed additional teacher input and studied the)A
36 333 :M
-.004(effect of the software by itself, but this would have given the students less support and in fact been unrealistic.)A
36 357 :M
-.007(We could have insisted on the tutoring as part of the EI being studied. This is sometimes justified as "using the)A
36 381 :M
-.006(software strictly in the way for which it was designed". But there are several problems with this: firstly, having)A
36 405 :M
-.001(a supervising teacher free for this is not something that could be guaranteed, so such a study would, like)A
36 429 :M
-.002(excluding tutoring, achieve uniform conditions at the expense of studying real conditions \(it would have)A
36 453 :M
-.001(required twice the staff: one to manage the lab, one to do the tutorials\). Secondly, in fact we would not have)A
36 477 :M
-.007(known that this was the appropriate condition to study, as neither the designers nor the teacher had planned this)A
36 501 :M
-.007(in advance: like a lot of teaching, such good practice is not a codified skill available for evaluators to consult)A
36 525 :M
-.002(when designing experiments, but rather something that may be observed and reported by them.)A
36 573 :M
-.005(Our studies, then, observe and can report real classroom practice. They are thus "ecologically valid". They)A
36 597 :M
-.001(cannot be regarded as well controlled experiments. On the other hand, although any such experiment might be)A
36 621 :M
-.007(replicable in another experiment, it would probably not predict what results would be obtained in real teaching.)A
36 645 :M
-.008(Our studies cannot be seen as observing "the effect of the software", but rather the combined effect of the)A
36 669 :M
-.002(overall teaching and learning situation. Complementary tutoring was one additional factor in this situation.)A
36 693 :M
-.007(Another mentioned earlier was external coercion to make students use the software at all \(a bigger factor in)A
36 717 :M
-.008(determining learning outcomes than any feature of the software design\). Some others should also be noted.)A
endp
%%Page: 16 16
%%BeginPageSetup
initializepage
(steve; page: 16 of 29)setjob
%%EndPageSetup
-31 -31 :T
gS 31 31 552 730 rC
36 44 :M
f0_9 sf
( )S
297 44 :M
( )S
549 44 :M
f1_9 sf
(16)S
36 69 :M
f0_12 sf
12 f4_1 :p
102.999 :m
-.034(Study habits for CAL)A
36 93 :M
-.001(In one study, a class was set to using some courseware. After watching them work for a bit, the teacher saw that)A
36 117 :M
-.009(very few notes were being taken and said "You could consider taking notes." Immediately all students began)A
36 141 :M
-.007(writing extensively. After a few minutes she couldn't bear it and said "You don't have to copy down every)A
36 165 :M
-.008(word", and again every students' behaviour changed. These students were in fact mature students, doing an)A
36 189 :M
-.001(education course, and in most matters less suggestible than typical students and more reflective about learning)A
36 213 :M
-.007(methods. Their extreme suggestibility here seems to reflect that, in contrast to lectures, say, there is no current)A
36 237 :M
-.002(student culture for learning from CAL from which they can draw study methods. The study habits they do)A
36 261 :M
-.007(employ are likely to have large effects on what they learn, but cannot be regarded as either a constant of the)A
36 285 :M
-.001(situation \(what students will normally do\) or as a personality characteristic. From the evaluation point of view)A
36 309 :M
-.002(it is an unstable factor. From the teaching viewpoint, it is probably something that needs active design and)A
36 333 :M
-.001(management. Again, it is a factor that neither designer nor teacher had planned in advance.)A
36 381 :M
87.999 :m
-.017(Hawthorne effects)A
36 405 :M
-.001(The Hawthorne effect is named after a study of work design in which whatever changes were made to the)A
36 429 :M
-.004(method of working, productivity went up, and the researcher concluded that the effect on the workers of being)A
36 453 :M
-.001(studied, and the concern and expectations that that implied, were having a bigger effect than changes to the)A
36 477 :M
-.005(method in themselves. Applied to studies of CAL, one might in principle distinguish between a Hawthorne)A
36 501 :M
-.004(effect of the CAL itself, and a Hawthorne effect of the evaluation. As to the latter, we have not seen any)A
36 525 :M
-.001(evidence of students being flattered or learning better just because we were observing them. In principle one)A
36 549 :M
-.005(could test this by comparing an obtrusive evaluation with one based only on exams and other class assessment,)A
36 573 :M
-.002(but we have not attempted this. Perhaps more interesting is the possibility, also not yet tested, that evaluation)A
36 597 :M
-.008(instruments such as confidence logs may enhance learning by promoting reflection and self-monitoring by)A
36 621 :M
-.001(students. Should this be the case, such instruments could easily become a permanent part of course materials)A
36 645 :M
-.004(\(like self-assessment questions\) to maintain the benefit.)A
36 693 :M
-.007(As to the effect from the CAL i.e. of learning being improved because students and perhaps teachers regard the)A
36 717 :M
-.001(CAL as glamourous advanced material, this is probably often the case, but we expect there equally to be cases)A
36 741 :M
-.001(of negative Hawthorne effects, as some groups of students regard it from the outset as a bad thing. We cannot)A
endp
%%Page: 17 17
%%BeginPageSetup
initializepage
(steve; page: 17 of 29)setjob
%%EndPageSetup
-31 -31 :T
gS 31 31 552 730 rC
36 44 :M
f0_9 sf
( )S
297 44 :M
( )S
549 44 :M
f1_9 sf
(17)S
36 69 :M
f0_12 sf
-.001(control or measure this effect precisely: unlike studies of medicines but like studies of surgical procedures, it is)A
36 93 :M
-.002(not possible to prevent subjects from knowing what "treatment" they are getting, and any psychological effects)A
36 117 :M
(of expectations to be activated. However it does mean that measuring students' attitudes to CAL should)S
36 141 :M
-.007(probably be done in every study, so that the likely direction of any effect is known. \(This implies, in fact, that)A
36 165 :M
-.004(the most impressive CAL is that which achieves good learning gains even with students who don't like it \321)A
36 189 :M
-.01(clearly that CAL is having more than a placebo effect.\))A
153 261 :M
f1_14 sf
-.016(5. Summary of the core attributes of our approach)A
36 285 :M
f0_12 sf
-.005(Our work began by trying to apply a wide range of instruments. We then went through a period of rapid)A
36 309 :M
-.001(adjustment to the needs and constraints of the situations in which our opportunities to evaluate occurred. In)A
36 333 :M
-.008(earlier sections we have described the components of our method, and then the issues which impressed)A
36 357 :M
-.007(themselves upon us as we adapted. What, in retrospect, are the essential features of our resulting method?)A
36 405 :M
-.008(Our approach is empirical: based on observing learning, not on judgements made by non-learners, expert or)A
36 429 :M
-.001(otherwise. Our studies have usually been felt to be useful by the teachers, even when we ourselves have felt)A
36 453 :M
-.001(that the issues were obvious and did not warrant the effort of a study: simply presenting the observations,)A
36 477 :M
-.005(measures, and collected student comments without any comment of our own has had a much greater power to)A
36 501 :M
-.005(convince teachers and start them actively changing things than any expert review would have had.)A
36 549 :M
-.001(This power of real observations applies equally to ourselves: we have in particular learned a great amount from)A
36 573 :M
-.001(open-ended measures and observations, which is how the unexpected can be detected. We therefore always)A
36 597 :M
-.004(include them, as well as planning to measure pre-identified issues. The chief of the latter are learning gains,)A
36 621 :M
-.004(which we always attempt to measure. Our measures for this \(confidence logs and quizzes\) are related directly)A
36 645 :M
-.012(to learning objectives, which gives these measures direct diagnostic power.)A
36 693 :M
-.001(Tests on CAL in any situation other than the intended teaching one are only weakly informative, hence our)A
36 717 :M
-.005(focus on classroom studies. This is because learning, especially in higher education, depends on the whole)A
36 741 :M
-.008(teaching and learning situation, comprising many factors, not just the "material" \(e.g. book or courseware\). You)A
endp
%%Page: 18 18
%%BeginPageSetup
initializepage
(steve; page: 18 of 29)setjob
%%EndPageSetup
-31 -31 :T
gS 31 31 552 730 rC
36 44 :M
f0_9 sf
( )S
297 44 :M
( )S
549 44 :M
f1_9 sf
(18)S
36 69 :M
f0_12 sf
-.008(have only to consider the effects on learning of leaving a book in a pub for "free access use", versus using it as a)A
36 93 :M
-.004(set text on a course, backed up by instructions on what to read, how it will be tested, and tutorials to answer)A
36 117 :M
-.001(questions. In any study there is a tension between the aims of science and education, between isolating factors)A
36 141 :M
-.001(and measuring their effects separately, and observing learning as it normally occurs with a view to maximising)A
36 165 :M
-.001(it in real situations. Our approach is closer to the latter emphasis. Partly because of this, these studies are)A
36 189 :M
-.007(crucially dependent on collaboration with the teacher, both for their tacit expertise in teaching and their explicit)A
36 213 :M
-.001(expertise in the subject being taught, which are both important in interpreting the observations and results.)A
36 261 :M
-.007(Ours is therefore distinct from various other approaches to evaluation. It differs from the checklist approach)A
36 285 :M
-.005(\(e.g. [8]\) because it is "student centered" in the sense that it is based on observation of what students actually do)A
36 309 :M
-.004(and feel. It differs from simply asking students their opinion of a piece of CAL because it attempts to measure)A
36 333 :M
-.007(learning, and to do this separately for each learning objective. It differs from simple pre- and post-testing)A
36 357 :M
-.004(designs because the substantial and systematic use of open-ended observation in real classroom situations often)A
36 381 :M
-.011(allows us to identify and report important factors that had not been foreseen.)A
36 429 :M
-.004(It also differs from production oriented evaluation, geared around the production of course material. These are)A
36 453 :M
-.004(similar in spirit to software engineering approaches in that they tend to assume that all the decisions about how)A
36 477 :M
-.008(teaching is delivered are "design decisions" and either are or should be taken in advance by a remote design)A
36 501 :M
-.004(team and held fixed once they have been tested and approved. In contrast our approach is to be prepared to)A
36 525 :M
-.001(observe and assist in the many local adaptations of how CAL is used that occur in practice, and to evaluate the)A
36 549 :M
-.004(local situation and practice as much as the CAL. It is our experience that designers of CAL frequently do not)A
36 573 :M
-.001(design in detail the teaching within which its use will be embedded, that teachers make many local adaptations)A
36 597 :M
-.001(just as they do for all other teaching, and that even if designers did prescribe use many teachers would still)A
36 621 :M
-.006(modify that use, just as they do that of textbooks.)A
36 669 :M
-.004(The food industry provides an analogy to this contrast. Prepared food ready for reheating is carefully designed)A
36 693 :M
-.002(to produce consistent results to which the end user contributes almost nothing. The instructions on the)A
36 717 :M
-.007(packaging attempt to ensure this by prescribing their actions. Production oriented evaluation is appropriate for)A
36 741 :M
-.001(this. In contrast, supplying ingredients \(meat, vegetables, etc.\) for home cooking involves rather different)A
endp
%%Page: 19 19
%%BeginPageSetup
initializepage
(steve; page: 19 of 29)setjob
%%EndPageSetup
-31 -31 :T
gS 31 31 552 730 rC
36 44 :M
f0_9 sf
( )S
297 44 :M
( )S
549 44 :M
f1_9 sf
(19)S
36 69 :M
f0_12 sf
-.002(issues. Clearly studies of how cooks use ingredients are important to improving supplies. However while some)A
36 93 :M
-.001(meals such as grilled fish and salads depend strongly on the quality of the ingredients, others such as pies, stews)A
36 117 :M
-.001(and curries were originally designed to make the best of low quality ingredients, and remain so successful that)A
36 141 :M
-.007(they are still widely valued. Such recipes are most easily varied for special dietry needs, most dependent on the)A
36 165 :M
-.01(cook's skills, and most easily adapted depending on what ingredients are available. Production oriented)A
36 189 :M
-.006(evaluation, organised as if there were one ideal form of the recipe, is not appropriate: rather an approach like)A
36 213 :M
-.018(ours that studies the whole situation is called for.)A
135 285 :M
f1_14 sf
-.01(6. Discussion: what is the use of such studies in practice?)A
36 309 :M
f0_12 sf
-.001(Having described and discussed many aspects of our approach, and having tried to summarise its core features,)A
36 333 :M
-.007(we now turn to the question of what, in retrospect, the uses of studies like ours turn out to be. We consider in)A
36 357 :M
-.002(turn five possible uses: formative, summative, illuminative, integrative, and QA functions.)A
36 405 :M
f1_12 sf
-.026(6.1 Formative evaluation)A
36 429 :M
f0_12 sf
-.003("Formative evaluation" is testing done with a view to modifying the software to solve any problems detected.)A
36 453 :M
-.004(Some of our studies have been formative, and contributed to improvements before release of the software.)A
36 477 :M
-.008(Because the aim is modification, the testing needs not only to detect the existence of a problem \(symptom)A
36 501 :M
-.008(detection\) but if possible to suggest what modification should be done: it needs to be diagnostic and suggestive)A
36 525 :M
-.001(of a remedy. Open-ended measures are therefore vital here to gather information about the nature of any)A
36 549 :M
-.005(problem \(e.g. do students misunderstand some item on the screen, or does it assume too much prior knowledge,)A
36 573 :M
(or what?\). However our learning measures \(quizzes and confidence logs\) are also useful here because, as they)S
36 597 :M
-.001(are mostly broken down into learning objectives, they are diagnostic and indicate with which objective the)A
36 621 :M
-.004(problem is occurring. In formative applications we might sharpen this further by asking students after each)A
36 645 :M
-.004(short section of the courseware \(which would usually correspond to a distinct learning objective\) to give ratings)A
36 669 :M
-.004(for the presentation, for the content, and for their confidence about a learning objective \(the one the designers)A
36 693 :M
-.003(thought corresponded to that section\), and also perhaps answer quiz items.)A
endp
%%Page: 20 20
%%BeginPageSetup
initializepage
(steve; page: 20 of 29)setjob
%%EndPageSetup
-31 -31 :T
gS 31 31 552 730 rC
36 44 :M
f0_9 sf
( )S
297 44 :M
( )S
549 44 :M
f1_9 sf
(20)S
36 69 :M
f0_12 sf
-.005(However many problems cannot be accurately detected without using realistic subjects and conditions, for)A
36 93 :M
-.007(reasons given earlier. Both apparent problems and apparent successes may change or disappear when real)A
36 117 :M
-.001(students with real motivations use the courseware. By that time however the software is by definition in use,)A
36 141 :M
-.002(and modifications to future versions of the software may not be the most important issue.)A
36 189 :M
f1_12 sf
-.013(6.2 Summative evaluation)A
36 213 :M
f0_12 sf
-.007("Summative evaluation" refers to studies done after implementation is complete, to sum up observed)A
36 237 :M
-.001(performance in some way, for instance consumer reports on microwave ovens. It could be used to inform)A
36 261 :M
-.007(decisions about which product to select, for example. They can be multi-dimensional in what they report, but in)A
36 285 :M
-.001(fact depend on there being a fairly narrow definition of how the device is to be used, and for what. As we have)A
36 309 :M
-.008(seen, this does not apply to the use of CAL in higher education. What matters most are the overall learning)A
36 333 :M
-.005(outcomes, but these are not determined only or mainly by the CAL: motivation, coercion, and other teaching)A
36 357 :M
-.001(and learning activities and resources are all crucial and these vary widely across situations. It is probably not)A
36 381 :M
-.004(sensible in practice to take the view that how a piece of courseware is used should be standardised, any more)A
36 405 :M
-.002(than a textbook should only be sold to teachers who promise to use it in a particular way.)A
36 453 :M
-.005(It is not sensible to design experiments to show whether CAL is better than lectures, any more than whether)A
36 477 :M
-.004(textbooks are good for learning: it all depends on the particular book, lecture, or piece of CAL. Slightly less)A
36 501 :M
-.005(obviously, it is not sensible to run experiments to test how effective a piece of CAL is, because learning is)A
36 525 :M
-.004(jointly determined by many factors and these vary widely across the situations that occur in practice. Well)A
36 549 :M
-.001(controlled experiments can be designed and run, but their results cannot predict effectiveness in any other)A
36 573 :M
-.004(situation than the artificial one tested because other factors have big effects. This means that we probably)A
36 597 :M
-.007(cannot even produce summative evaluations as systematic as consumer reports on ovens. Ovens are effectively)A
36 621 :M
-.004(the sole cause of cooking for food placed in them, while CAL is probably more like the role of an open window)A
36 645 :M
-.002(in cooling a room in summer: crucial, but with effects that depend as much on the situation as they do on the)A
36 669 :M
-.032(design of the window.)A
36 717 :M
-.002(However this does not mean that no useful summative evaluation is possible. When you are selecting a)A
36 741 :M
-.007(textbook or piece of CAL for use you may have to do this on the basis of a few reviews, but you would certainly)A
endp
%%Page: 21 21
%%BeginPageSetup
initializepage
(steve; page: 21 of 29)setjob
%%EndPageSetup
-31 -31 :T
gS 31 31 552 730 rC
36 44 :M
f0_9 sf
( )S
297 44 :M
( )S
549 44 :M
f1_9 sf
(21)S
36 69 :M
f0_12 sf
-.007(like to hear that someone had used it in earnest on a class, and how that turned out. The more the situation is)A
36 93 :M
-.001(like your own, the more detailed measures are reported, and the more issues \(e.g. need for tutorial support\))A
36 117 :M
-.004(identified as critical, the more useful such a report would be. In this weak but important sense, summative)A
36 141 :M
-.002(evaluations of CAL are useful. Many of our studies have performed summatively in this way, allowing)A
36 165 :M
-.008(developers to show enquirers that the software has been used and tested, and with substantial details of its)A
36 189 :M
-.056(performance.)A
36 237 :M
f1_12 sf
-.037(6.3 Illuminative evaluation)A
36 261 :M
f0_12 sf
-.003("Illuminative evaluation" is a term introduced by Parlett & Hamilton [9, 10] to denote an observational)A
36 285 :M
-.001(approach inspired by ethnographic rather than experimental traditions and methods. Its aim is to discover, not)A
36 309 :M
-.004(how an EI \(educational intervention\) performs on standard measures, but what the factors and issues are that are)A
36 333 :M
-.001(important to the participants in that particular situation, or which seem evidently crucial to a close observer.)A
36 381 :M
-.001(The importance of this has certainly impressed itelf upon us, leading to our stress on the open-ended methods)A
36 405 :M
-.007(that have taught us so much. In particular, they allow us to identify and report on factors important in particular)A
36 429 :M
-.007(cases, which is an important aspect of our summative reports, given the situation-dependent nature of learning.)A
36 453 :M
-.001(They have also allowed us to identify factors that are probably of wider importance, such as the instability but)A
36 477 :M
-.003(importance of the study methods that students bring to bear on CAL material.)A
36 525 :M
-.008(Thus our studies have an important illuminative aspect and function, although they combine it with systematic)A
36 549 :M
-.005(comparative methods, as Parlett & Hamilton originally recommended. Whether we have achieved the right)A
36 573 :M
-.015(balance or compromise is harder to judge.)A
36 621 :M
f1_12 sf
-.037(6.4 Integrative evaluation)A
36 645 :M
f0_12 sf
-.005(Although our studies can and have performed the traditional formative and summative functions discussed)A
36 669 :M
-.001(above, in a number of cases we have seen a new role for them emerge of perhaps greater benefit. The)A
36 693 :M
-.001(experience for a typical client of ours \(i.e. someone responsible for a course\) is of initially low expectations of)A
36 717 :M
-.001(usefulness for the evaluation \321 after all, if they had serious doubts about whether the planned teaching was)A
36 741 :M
-.004(satisfactory they would already have started to modify it. However when they see the results of our evaluation,)A
endp
%%Page: 22 22
%%BeginPageSetup
initializepage
(steve; page: 22 of 29)setjob
%%EndPageSetup
-31 -31 :T
gS 31 31 552 730 rC
36 44 :M
f0_9 sf
( )S
297 44 :M
( )S
549 44 :M
f1_9 sf
(22)S
36 69 :M
f0_12 sf
-.005(they may find on the one hand confirmation that many of their objectives are indeed being satisfactorily)A
36 93 :M
-.001(achieved \(and now they have better evidence of this than before\), and on the other hand that some did)A
36 117 :M
-.008(unexpectedly poorly \321 but that they can immediately think of ways to tackle this by adjusting aspects of the)A
36 141 :M
-.007(delivered teaching without large costs in effort or other resources. For example, a particular item shown to be)A
36 165 :M
-.007(unsuccessfully handled by the software alone might be specifically addressed in a lecture, supplemented by a)A
36 189 :M
-.007(new handout, or become the focus of companion tutorials. This is not very different from the way that teachers)A
36 213 :M
-.007(dynamically adjust their teaching in the light of other feedback e.g. audience reaction, and is a strength of any)A
36 237 :M
-.004(face to face course where at least some elements \(e.g. what is said, handouts, new OHP slides\) can be modified)A
36 261 :M
-.004(quickly. The difference is in the quality of the feedback information. Because our approach to evaluation is)A
36 285 :M
-.004(based around each teacher's own statement of learning objectives, and around the teacher's own test items, the)A
36 309 :M
-.007(results are directly relevant to how the teacher thinks about the course and what they are trying to achieve: so it)A
36 333 :M
-.02(is not surprising that teachers find it useful.)A
36 381 :M
-.008(An example of this is a recent study of automated psychology labs, where students work their way through a)A
36 405 :M
-.001(series of computer-mediated experiments and complete reports in an accompanying booklet. In fact the)A
36 429 :M
-.004(objectives addressed by the software were all performing well, but the evaluation showed that the weakest point)A
36 453 :M
-.002(was in writing the discussion section of the reports, which was not properly supported. This focussed diagnosis)A
36 477 :M
-.001(immediately suggested \(to the teacher\) a remedy that will be now be acted on \(a new specialised worksheet and)A
36 501 :M
-.004(a new topic for tutorials\), where previous generalised complaints had not been taken as indicating a fault with)A
36 525 :M
-.026(the teaching.)A
36 573 :M
-.001(How does this kind of evaluation fit in with the other kinds of available feedback about teaching? The oldest)A
36 597 :M
-.002(and probably most trusted kind of feedback in university teaching is direct verbal questions, complaints, and)A
36 621 :M
-.002(comments from students to teachers. Undoubtedly many local problems are quickly and dynamically adjusted)A
36 645 :M
-.001(in this way. Its disadvantages, which grow greater with larger class sizes, are that the teacher cannot tell how)A
36 669 :M
-.006(representative of the class each comment is, and that obviously "typical" students do not comment because only)A
36 693 :M
-.004(a very small, self-selected, number of students get to do this. Course feedback questionnaires get round this)A
36 717 :M
-.004(problem of representativeness by getting feedback from all students. However they are generally used only)A
36 741 :M
-.007(once a term, and so are usually designed to ask about general aspects of the course, such as whether the lecturer)A
endp
%%Page: 23 23
%%BeginPageSetup
initializepage
(steve; page: 23 of 29)setjob
%%EndPageSetup
-31 -31 :T
gS 31 31 552 730 rC
36 44 :M
f0_9 sf
( )S
297 44 :M
( )S
549 44 :M
f1_9 sf
(23)S
36 69 :M
f0_12 sf
-.001(is enthusiastic, well-organised, and so on. It is not easy for teachers to see how to change their behaviour to)A
36 93 :M
-.007(affect such broad issues, which are certainly not directly about specific content \(which after all is the whole)A
36 117 :M
-.007(point of the course\), how well it is being learned, and how teachers could support that learning better. Our)A
36 141 :M
-.004(methods are more detailed, but crucially they are much more diagnostic: much more likely to make it easy for)A
36 165 :M
-.003(teachers to think of a relevant change to address the problem.)A
36 213 :M
-.006(This constitutes a new role for evaluation that may be called "integrative": evaluation aimed at improving)A
36 237 :M
-.007(teaching and learning by better integration of the CAL material into the overall situation. It is not primarily)A
36 261 :M
-.008(either formative or summative of the software, as what is both measured and modified is most often not the)A
36 285 :M
-.004(software but surrounding materials and activities. It is not merely reporting on measurements as summative)A
36 309 :M
-.007(evaluation is, because it typically leads to immediate action in the form of changes. It could therefore be called)A
36 333 :M
-.005(formative evaluation of the overall teaching situation, but we call it "integrative" to suggest the nature of the)A
36 357 :M
-.007(changes it leads to. This role for evaluation is compatible with the issues that are problems for the role of)A
36 381 :M
-.005(summative evaluation, such as observing only whole situations and the net effect of many influences on)A
36 405 :M
-.004(learning. After all, that is what teachers are in fact really concerned with: not the properties of CAL, but the)A
36 429 :M
-.01(delivery of effective teaching and learning using CAL.)A
36 477 :M
f1_12 sf
-.02(6.5 QA functions)A
36 501 :M
f0_12 sf
-.008(Such integrative evaluation can also be useful in connection with the QA \(quality audit, assessment, or)A
36 525 :M
-.008(assurance\) procedures being introduced in UK universities, and in fact can equally be applied to non-CAL)A
36 549 :M
-.005(teaching. Firstly it provides much better than normal evidence about quality already achieved. Secondly it)A
36 573 :M
-.007(demonstrates that quality is being actively monitored using extensive student-based measures. Thirdly, since it)A
36 597 :M
-.001(usually leads to modifications by the teachers without any outside prompting, it provides evidence of teachers)A
36 621 :M
-.001(acting on results to improve quality. Thus performing integrative evaluations can leave the teachers in the)A
36 645 :M
-.002(position of far exceeding current QA standards, while improving their teaching in their own terms.)A
36 693 :M
-.004(These advantages stem from our adoption of the same objectives-based approach that the positive side of QA is)A
36 717 :M
-.001(concerned with. In practice it can have the further advantage that teachers can use the evaluation episode to)A
36 741 :M
-.007(work up the written statement of their objectives in a context where this brings them some direct benefit, and)A
endp
%%Page: 24 24
%%BeginPageSetup
initializepage
(steve; page: 24 of 29)setjob
%%EndPageSetup
-31 -31 :T
gS 31 31 552 730 rC
36 44 :M
f0_9 sf
( )S
297 44 :M
( )S
549 44 :M
f1_9 sf
(24)S
36 69 :M
f0_12 sf
-.007(where the statements get debugged in the attempt to associate them with test items. They can then re-use them)A
36 93 :M
-.007(for any QA paperwork they may be asked for at some other time. This can overcome the resistance many feel)A
36 117 :M
-.008(when objectives appear as a paper exercise divorced from any function contributing to the teaching.)A
36 165 :M
f1_12 sf
-.014(6.6 Limitations and the need for future work)A
36 189 :M
f0_12 sf
-.004(Our approach has various limitations associated with the emphasis on particular classroom episodes. We have)A
36 213 :M
-.002(not developed convincing tests of deep as opposed to shallow learning [11]: of understanding as opposed to the)A
36 237 :M
-.004(ability to answer short quiz items. Thus we have almost always looked at many small learning objectives, rather)A
36 261 :M
-.001(than how to test for large ones concerning the understanding of central but complex concepts. This should not)A
36 285 :M
-.002(be incompatible with our approach, but will require work to enable us to suggest to our clients how to develop)A
36 309 :M
-.008(such test items. Similarly we have considered, but not achieved, measures of a student's effective intention in a)A
36 333 :M
-.005(given learning situation \(their "task grasp"\) which probably determines whether they do deep learning, shallow)A
36 357 :M
-.004(learning or no learning. For instance, when a student is working through a lab class is she essentially acting to)A
36 381 :M
-.002(get through the afternoon, to complete the worksheet, to get the "right" result, or to explore scientific concepts?)A
36 405 :M
-.005(The same issue is important in CAL, and probably determines whether students flip through the screens, read)A
36 429 :M
-.001(them through once expecting learning to just happen, or actively engage in some definite agenda of their own.)A
36 477 :M
-.002(It is also important to keep in mind a more profound limitation of the scope of such studies: they are about how)A
36 501 :M
-.004(particular teaching materials perform, and how to adjust the overall situation to improve learning. Such studies)A
36 525 :M
-.008(are unlikely to initiate big changes and perhaps big improvements such as a shift from topic-based to problem)A
563 525 :M
(-)S
36 549 :M
-.001(based learning, or the abandonment of lectures in favour of other learning activities. They will not replace the)A
36 573 :M
-.001(research that goes into important educational advances, although they can, we believe, be useful in making the)A
36 597 :M
-.007(best of such advances by adjusting their application when they are introduced. Integrative evaluation is likely to)A
36 621 :M
-.011(promote small and local evolutionary adaptations, not revolutionary advances.)A
36 669 :M
f1_12 sf
-.02(Acknowledgements)A
36 693 :M
f0_12 sf
-.001(This work was done as part of the TILT \(Teaching with Independent Learning Technologies\) project, funded)A
36 717 :M
-.008(through the TLTP programme \(Teaching and Learning Technology Programme\) by the UK university funding)A
36 741 :M
-.008(bodies \(DENI, HEFCE, HEFCW, SHEFC\) and by the University of Glasgow. The studies discussed here could)A
endp
%%Page: 25 25
%%BeginPageSetup
initializepage
(steve; page: 25 of 29)setjob
%%EndPageSetup
-31 -31 :T
gS 31 31 552 730 rC
36 44 :M
f0_9 sf
( )S
297 44 :M
( )S
549 44 :M
f1_9 sf
(25)S
36 69 :M
f0_12 sf
-.001(not have been done without the active participation of many members of the university teaching staff to whom)A
36 93 :M
-.063(we are grateful.)A
273 189 :M
f1_14 sf
-.031(References)A
36 213 :M
f0_12 sf
(1.)S
58 213 :M
-.005(Doughty,G., Arnold,S., Barr,N., Brown,M.I., Creanor,L., Donnelly,P.J., Draper,S.W., Duffy,C.,)A
58 237 :M
-.008(Durndell,H., Harrison,M., Henderson,F.P., Jessop,A., McAteer,E., Milner,M., Neil,D.M., Pflicke,T.,)A
58 261 :M
-.008(Pollock,M., Primrose,C., Richard,S., Sclater,N., Shaw,R., Tickner,S., Turner,I., van der Zwan,R. &)A
58 285 :M
-.065(Watt,H.D. )A
114 285 :M
12 f4_1 :p
343.4 :m
-.016(Using learning technologies: interim conclusions from the TILT project)A
-.016( TILT project report)A
58 309 :M
-.011(no.3, Robert Clark Centre, University of Glasgow \(1995\))A
36 357 :M
(2.)S
58 357 :M
-.009(Draper,S.W., Brown,M.I., Edgerton,E., Henderson,F.P., McAteer,E., Smith,E.D., & Watt,H.D. )A
523 357 :M
49 :m
-.123(Observing)A
58 381 :M
278.962 :m
-.004(and measuring the performance of educational technology)A
-.004( TILT project report no.1, Robert Clark Centre,)A
58 405 :M
-.011(University of Glasgow \(1994\))A
36 453 :M
(3.)S
58 453 :M
-.005(Creanor,L., Durndell,H., Henderson,F.P., Primrose,C., Brown,M.I., Draper,S.W., McAteer,E. )A
529 453 :M
8.663 :m
(A)S
58 477 :M
333.674 :m
-.008(hypertext approach to information skills: development and evaluation)A
-.008( TILT project report no.4, Robert)A
58 501 :M
-.023(Clark Centre, University of Glasgow \(1995\))A
36 549 :M
(4.)S
58 549 :M
-.003(Brown,M.I., Doughty,G.F., Draper,S.W., Henderson,F.P., McAteer,E. "Measuring learning resource use")A
58 573 :M
-.038(submitted to this issue of )A
181 573 :M
116 :m
-.032(Computers & Education)A
298 573 :M
-.164( \(1996\))A
36 621 :M
(5.)S
58 621 :M
-.001(McAteer,E., Brown,M.I., Draper,S.W., Henderson,F.P., Barr,N. & Neil,D. "Simulation software in a)A
58 645 :M
-.012(life sciences practical laboratory" submitted to this issue of )A
116.329 :m
-.016(Computers & Education)A
464 645 :M
-.166( \(1996)A
498 645 :M
3.993 :m
(\))S
36 693 :M
(6.)S
58 693 :M
(Henderson,F.P., Creanor,L., Duffy,C. & Tickner,S. "Case studies in evaluation" submitted to this issue)S
58 717 :M
-.497(of )A
71 717 :M
116 :m
-.032(Computers & Education)A
188 717 :M
-.164( \(1996\))A
endp
%%Page: 26 26
%%BeginPageSetup
initializepage
(steve; page: 26 of 29)setjob
%%EndPageSetup
-31 -31 :T
gS 31 31 552 730 rC
36 44 :M
f0_9 sf
( )S
297 44 :M
( )S
549 44 :M
f1_9 sf
(26)S
36 69 :M
f0_12 sf
(7.)S
58 69 :M
-.005(Marsh, H.W. "Student's evaluations of university teaching: research findings, methodological issues, and)A
58 93 :M
-.025(directions for future research" )A
208 93 :M
12 f4_1 :p
167.109 :m
-.013(Int. journal of educational research)A
-.014( vol.11 no.3 pp.253-388 \(1987\))A
36 141 :M
(8.)S
58 141 :M
-.028(Machell, J. & Saunders,M. \(eds.\) )A
226 141 :M
232.001 :m
-.013(MEDA: An evaluation tool for training software)A
459 141 :M
-.026( Centre for the study of)A
58 165 :M
-.01(education and training, University of Lancaster \(1991\))A
36 213 :M
(9.)S
58 213 :M
-.004(Parlett, M.R. & Hamilton,D. "Evaluation as illumination: a new approach to the study of innovatory)A
58 237 :M
-.079(programmes".)A
58 261 :M
-.007(Workshop at Cambridge, and unpublished report Occasional paper 9, Centre for research in the educational)A
58 285 :M
-.006(sciences, University of Edinburgh. \(1972\))A
58 309 :M
-.008(And in D.Hamilton, D.Jenkins, C.King, B.MacDonald & M.Parlett \(eds.\) )A
428 309 :M
141.999 :m
-.037(Beyond the numbers game: a)A
58 333 :M
153 :m
-.019(reader in educational evaluation)A
212 333 :M
-.005( \(Basingstoke: Macmillan\) ch.1.1 pp.6-22. \(1977\))A
58 357 :M
-.025(And in R.Murphy & H.Torrance \(eds.\) )A
253 357 :M
200.001 :m
-.016(Evaluating education: issues and methods)A
454 357 :M
-.046( \(Milton Keynes: Open)A
58 381 :M
-.007(University Press\) ch.1.4 pp.57-73 \(1987\))A
36 429 :M
(10.)S
58 429 :M
-.025(Parlett, M. & Dearden,G. )A
186 429 :M
319.999 :m
-.014(Introduction to illuminative evaluation: studies in higher education)A
507 429 :M
-.034( \(Pacific)A
58 453 :M
-.013(soundings press\) \(1977\))A
36 501 :M
(11.)S
58 501 :M
-.015(Marton,F., Hounsell,D. & Entwistle,N. \(eds.\) )A
128.496 :m
-.017(The experience of learning)A
422 501 :M
-.032( \(Edinburgh: Scottish academic)A
58 525 :M
-.053(press\) \(1984\))A
endp
%%Page: 27 27
%%BeginPageSetup
initializepage
(steve; page: 27 of 29)setjob
%%EndPageSetup
-31 -31 :T
gS 31 31 552 730 rC
36 44 :M
f0_9 sf
( )S
297 44 :M
( )S
549 44 :M
f1_9 sf
(27)S
210 69 :M
f1_12 sf
-.009(Appendix A: Student confidence logs)A
36 141 :M
-.006(Please indicate, by ticking the relevant box, how confident you feel that you are able to:)A
1 G
0 0 1 1 rF
481 141 :M
psb
/wp$gray 0.8750 def
/wp$top 165 def
/wp$left 84 def
/wp$right 191 def
/wp$bottom 183 def
/wp$box {newpath wp$left wp$bottom moveto wp$left wp$top lineto wp$right wp$top lineto wp$right wp$bottom lineto closepath } def
wp$gray setgray wp$box fill
pse
0 0 1 1 rF
36 159 :M
psb
/wp$gray 0.8750 def
/wp$top 183 def
/wp$left 84 def
/wp$right 191 def
/wp$bottom 201 def
/wp$box {newpath wp$left wp$bottom moveto wp$left wp$top lineto wp$right wp$top lineto wp$right wp$bottom lineto closepath } def
wp$gray setgray wp$box fill
pse
123 198 :M
0 G
-.084(Topic)A
1 G
0 0 1 1 rF
152 198 :M
psb
/wp$gray 0.8750 def
/wp$top 201 def
/wp$left 84 def
/wp$right 191 def
/wp$bottom 219 def
/wp$box {newpath wp$left wp$bottom moveto wp$left wp$top lineto wp$right wp$top lineto wp$right wp$bottom lineto closepath } def
wp$gray setgray wp$box fill
pse
0 0 1 1 rF
152 198 :M
psb
/wp$gray 0.8750 def
/wp$top 165 def
/wp$left 191 def
/wp$right 252 def
/wp$bottom 183 def
/wp$box {newpath wp$left wp$bottom moveto wp$left wp$top lineto wp$right wp$top lineto wp$right wp$bottom lineto closepath } def
wp$gray setgray wp$box fill
pse
210 180 :M
0 G
-.105(Very)A
1 G
0 0 1 1 rF
235 180 :M
psb
/wp$gray 0.8750 def
/wp$top 183 def
/wp$left 191 def
/wp$right 252 def
/wp$bottom 201 def
/wp$box {newpath wp$left wp$bottom moveto wp$left wp$top lineto wp$right wp$top lineto wp$right wp$bottom lineto closepath } def
wp$gray setgray wp$box fill
pse
197 198 :M
0 G
-.04(Confident)A
1 G
0 0 1 1 rF
248 198 :M
psb
/wp$gray 0.8750 def
/wp$top 201 def
/wp$left 191 def
/wp$right 252 def
/wp$bottom 219 def
/wp$box {newpath wp$left wp$bottom moveto wp$left wp$top lineto wp$right wp$top lineto wp$right wp$bottom lineto closepath } def
wp$gray setgray wp$box fill
pse
0 0 1 1 rF
248 198 :M
psb
/wp$gray 0.8750 def
/wp$top 165 def
/wp$left 252 def
/wp$right 313 def
/wp$bottom 183 def
/wp$box {newpath wp$left wp$bottom moveto wp$left wp$top lineto wp$right wp$top lineto wp$right wp$bottom lineto closepath } def
wp$gray setgray wp$box fill
pse
0 0 1 1 rF
248 198 :M
psb
/wp$gray 0.8750 def
/wp$top 183 def
/wp$left 252 def
/wp$right 313 def
/wp$bottom 201 def
/wp$box {newpath wp$left wp$bottom moveto wp$left wp$top lineto wp$right wp$top lineto wp$right wp$bottom lineto closepath } def
wp$gray setgray wp$box fill
pse
258 198 :M
0 G
-.04(Confident)A
1 G
0 0 1 1 rF
309 198 :M
psb
/wp$gray 0.8750 def
/wp$top 201 def
/wp$left 252 def
/wp$right 313 def
/wp$bottom 219 def
/wp$box {newpath wp$left wp$bottom moveto wp$left wp$top lineto wp$right wp$top lineto wp$right wp$bottom lineto closepath } def
wp$gray setgray wp$box fill
pse
0 0 1 1 rF
309 198 :M
psb
/wp$gray 0.8750 def
/wp$top 165 def
/wp$left 313 def
/wp$right 382 def
/wp$bottom 183 def
/wp$box {newpath wp$left wp$bottom moveto wp$left wp$top lineto wp$right wp$top lineto wp$right wp$bottom lineto closepath } def
wp$gray setgray wp$box fill
pse
334 180 :M
0 G
-.33(Some)A
1 G
0 0 1 1 rF
361 180 :M
psb
/wp$gray 0.8750 def
/wp$top 183 def
/wp$left 313 def
/wp$right 382 def
/wp$bottom 201 def
/wp$box {newpath wp$left wp$bottom moveto wp$left wp$top lineto wp$right wp$top lineto wp$right wp$bottom lineto closepath } def
wp$gray setgray wp$box fill
pse
319 198 :M
0 G
-.109(Confidence)A
1 G
0 0 1 1 rF
376 198 :M
psb
/wp$gray 0.8750 def
/wp$top 201 def
/wp$left 313 def
/wp$right 382 def
/wp$bottom 219 def
/wp$box {newpath wp$left wp$bottom moveto wp$left wp$top lineto wp$right wp$top lineto wp$right wp$bottom lineto closepath } def
wp$gray setgray wp$box fill
pse
0 0 1 1 rF
377 198 :M
psb
/wp$gray 0.8750 def
/wp$top 165 def
/wp$left 382 def
/wp$right 450 def
/wp$bottom 183 def
/wp$box {newpath wp$left wp$bottom moveto wp$left wp$top lineto wp$right wp$top lineto wp$right wp$bottom lineto closepath } def
wp$gray setgray wp$box fill
pse
403 180 :M
0 G
-.197(Little)A
1 G
0 0 1 1 rF
430 180 :M
psb
/wp$gray 0.8750 def
/wp$top 183 def
/wp$left 382 def
/wp$right 450 def
/wp$bottom 201 def
/wp$box {newpath wp$left wp$bottom moveto wp$left wp$top lineto wp$right wp$top lineto wp$right wp$bottom lineto closepath } def
wp$gray setgray wp$box fill
pse
388 198 :M
0 G
-.109(Confidence)A
1 G
0 0 1 1 rF
445 198 :M
psb
/wp$gray 0.8750 def
/wp$top 201 def
/wp$left 382 def
/wp$right 450 def
/wp$bottom 219 def
/wp$box {newpath wp$left wp$bottom moveto wp$left wp$top lineto wp$right wp$top lineto wp$right wp$bottom lineto closepath } def
wp$gray setgray wp$box fill
pse
0 0 1 1 rF
446 198 :M
psb
/wp$gray 0.8750 def
/wp$top 165 def
/wp$left 450 def
/wp$right 521 def
/wp$bottom 183 def
/wp$box {newpath wp$left wp$bottom moveto wp$left wp$top lineto wp$right wp$top lineto wp$right wp$bottom lineto closepath } def
wp$gray setgray wp$box fill
pse
479 180 :M
0 G
-.663(No)A
1 G
0 0 1 1 rF
493 180 :M
psb
/wp$gray 0.8750 def
/wp$top 183 def
/wp$left 450 def
/wp$right 521 def
/wp$bottom 201 def
/wp$box {newpath wp$left wp$bottom moveto wp$left wp$top lineto wp$right wp$top lineto wp$right wp$bottom lineto closepath } def
wp$gray setgray wp$box fill
pse
458 198 :M
0 G
-.109(Confidence)A
1 G
0 0 1 1 rF
515 198 :M
psb
/wp$gray 0.8750 def
/wp$top 201 def
/wp$left 450 def
/wp$right 521 def
/wp$bottom 219 def
/wp$box {newpath wp$left wp$bottom moveto wp$left wp$top lineto wp$right wp$top lineto wp$right wp$bottom lineto closepath } def
wp$gray setgray wp$box fill
pse
456 216 :M
0 G
-.034(Whatsoever)A
82 163 -1 1 85 162 1 82 162 @a
-1 -1 83 165 1 1 82 162 @b
-1 -1 85 165 1 1 84 164 @b
-1 -1 85 165 1 1 84 164 @b
85 163 -1 1 191 162 1 85 162 @a
85 165 -1 1 191 164 1 85 164 @a
191 163 -1 1 194 162 1 191 162 @a
191 165 -1 1 194 164 1 191 164 @a
194 163 -1 1 252 162 1 194 162 @a
194 165 -1 1 252 164 1 194 164 @a
252 163 -1 1 255 162 1 252 162 @a
252 165 -1 1 255 164 1 252 164 @a
255 163 -1 1 313 162 1 255 162 @a
255 165 -1 1 313 164 1 255 164 @a
313 163 -1 1 316 162 1 313 162 @a
313 165 -1 1 316 164 1 313 164 @a
316 163 -1 1 382 162 1 316 162 @a
316 165 -1 1 382 164 1 316 164 @a
382 163 -1 1 385 162 1 382 162 @a
382 165 -1 1 385 164 1 382 164 @a
385 163 -1 1 450 162 1 385 162 @a
385 165 -1 1 450 164 1 385 164 @a
450 163 -1 1 453 162 1 450 162 @a
450 165 -1 1 453 164 1 450 164 @a
453 163 -1 1 521 162 1 453 162 @a
453 165 -1 1 521 164 1 453 164 @a
521 163 -1 1 524 162 1 521 162 @a
-1 -1 524 165 1 1 523 162 @b
-1 -1 522 165 1 1 521 164 @b
-1 -1 522 165 1 1 521 164 @b
-1 -1 83 219 1 1 82 165 @b
-1 -1 85 219 1 1 84 165 @b
-1 -1 192 219 1 1 191 165 @b
-1 -1 253 219 1 1 252 165 @b
-1 -1 314 219 1 1 313 165 @b
-1 -1 383 219 1 1 382 165 @b
-1 -1 451 219 1 1 450 165 @b
-1 -1 522 219 1 1 521 165 @b
-1 -1 524 219 1 1 523 165 @b
88 235 :M
f0_12 sf
-.018(a\) Sequence text)A
88 253 :M
-.111(with sound)A
1 G
0 0 1 1 rF
140 253 :M
psb
/wp$gray 1 def
/wp$top 238 def
/wp$left 191 def
/wp$right 252 def
/wp$bottom 256 def
/wp$box {newpath wp$left wp$bottom moveto wp$left wp$top lineto wp$right wp$top lineto wp$right wp$bottom lineto closepath } def
wp$gray setgray wp$box fill
pse
0 0 1 1 rF
196 235 :M
psb
/wp$gray 1 def
/wp$top 238 def
/wp$left 252 def
/wp$right 313 def
/wp$bottom 256 def
/wp$box {newpath wp$left wp$bottom moveto wp$left wp$top lineto wp$right wp$top lineto wp$right wp$bottom lineto closepath } def
wp$gray setgray wp$box fill
pse
0 0 1 1 rF
257 235 :M
psb
/wp$gray 1 def
/wp$top 238 def
/wp$left 313 def
/wp$right 382 def
/wp$bottom 256 def
/wp$box {newpath wp$left wp$bottom moveto wp$left wp$top lineto wp$right wp$top lineto wp$right wp$bottom lineto closepath } def
wp$gray setgray wp$box fill
pse
0 0 1 1 rF
318 235 :M
psb
/wp$gray 1 def
/wp$top 238 def
/wp$left 382 def
/wp$right 450 def
/wp$bottom 256 def
/wp$box {newpath wp$left wp$bottom moveto wp$left wp$top lineto wp$right wp$top lineto wp$right wp$bottom lineto closepath } def
wp$gray setgray wp$box fill
pse
0 0 1 1 rF
387 235 :M
psb
/wp$gray 1 def
/wp$top 238 def
/wp$left 450 def
/wp$right 521 def
/wp$bottom 256 def
/wp$box {newpath wp$left wp$bottom moveto wp$left wp$top lineto wp$right wp$top lineto wp$right wp$bottom lineto closepath } def
wp$gray setgray wp$box fill
pse
0 G
-1 -1 85 220 1 1 84 219 @b
-1 -1 83 220 1 1 82 219 @b
85 220 -1 1 191 219 1 85 219 @a
-1 -1 192 220 1 1 191 219 @b
192 220 -1 1 252 219 1 192 219 @a
-1 -1 253 220 1 1 252 219 @b
253 220 -1 1 313 219 1 253 219 @a
-1 -1 314 220 1 1 313 219 @b
314 220 -1 1 382 219 1 314 219 @a
-1 -1 383 220 1 1 382 219 @b
383 220 -1 1 450 219 1 383 219 @a
-1 -1 451 220 1 1 450 219 @b
451 220 -1 1 521 219 1 451 219 @a
-1 -1 524 220 1 1 523 219 @b
-1 -1 522 220 1 1 521 219 @b
-1 -1 83 256 1 1 82 220 @b
-1 -1 85 256 1 1 84 220 @b
-1 -1 192 256 1 1 191 220 @b
-1 -1 253 256 1 1 252 220 @b
-1 -1 314 256 1 1 313 220 @b
-1 -1 383 256 1 1 382 220 @b
-1 -1 451 256 1 1 450 220 @b
-1 -1 522 256 1 1 521 220 @b
-1 -1 524 256 1 1 523 220 @b
88 272 :M
-.048(b\) Accurately)A
88 290 :M
-.051(identify the keyword)A
88 308 :M
-.016(in a given statement)A
88 326 :M
-.065(or question)A
1 G
0 0 1 1 rF
141 326 :M
psb
/wp$gray 1 def
/wp$top 275 def
/wp$left 191 def
/wp$right 252 def
/wp$bottom 329 def
/wp$box {newpath wp$left wp$bottom moveto wp$left wp$top lineto wp$right wp$top lineto wp$right wp$bottom lineto closepath } def
wp$gray setgray wp$box fill
pse
0 0 1 1 rF
196 272 :M
psb
/wp$gray 1 def
/wp$top 275 def
/wp$left 252 def
/wp$right 313 def
/wp$bottom 329 def
/wp$box {newpath wp$left wp$bottom moveto wp$left wp$top lineto wp$right wp$top lineto wp$right wp$bottom lineto closepath } def
wp$gray setgray wp$box fill
pse
0 0 1 1 rF
257 272 :M
psb
/wp$gray 1 def
/wp$top 275 def
/wp$left 313 def
/wp$right 382 def
/wp$bottom 329 def
/wp$box {newpath wp$left wp$bottom moveto wp$left wp$top lineto wp$right wp$top lineto wp$right wp$bottom lineto closepath } def
wp$gray setgray wp$box fill
pse
0 0 1 1 rF
318 272 :M
psb
/wp$gray 1 def
/wp$top 275 def
/wp$left 382 def
/wp$right 450 def
/wp$bottom 329 def
/wp$box {newpath wp$left wp$bottom moveto wp$left wp$top lineto wp$right wp$top lineto wp$right wp$bottom lineto closepath } def
wp$gray setgray wp$box fill
pse
0 0 1 1 rF
387 272 :M
psb
/wp$gray 1 def
/wp$top 275 def
/wp$left 450 def
/wp$right 521 def
/wp$bottom 329 def
/wp$box {newpath wp$left wp$bottom moveto wp$left wp$top lineto wp$right wp$top lineto wp$right wp$bottom lineto closepath } def
wp$gray setgray wp$box fill
pse
0 G
-1 -1 85 257 1 1 84 256 @b
-1 -1 83 257 1 1 82 256 @b
85 257 -1 1 191 256 1 85 256 @a
-1 -1 192 257 1 1 191 256 @b
192 257 -1 1 252 256 1 192 256 @a
-1 -1 253 257 1 1 252 256 @b
253 257 -1 1 313 256 1 253 256 @a
-1 -1 314 257 1 1 313 256 @b
314 257 -1 1 382 256 1 314 256 @a
-1 -1 383 257 1 1 382 256 @b
383 257 -1 1 450 256 1 383 256 @a
-1 -1 451 257 1 1 450 256 @b
451 257 -1 1 521 256 1 451 256 @a
-1 -1 524 257 1 1 523 256 @b
-1 -1 522 257 1 1 521 256 @b
-1 -1 83 329 1 1 82 257 @b
-1 -1 85 329 1 1 84 257 @b
-1 -1 192 329 1 1 191 257 @b
-1 -1 253 329 1 1 252 257 @b
-1 -1 314 329 1 1 313 257 @b
-1 -1 383 329 1 1 382 257 @b
-1 -1 451 329 1 1 450 257 @b
-1 -1 522 329 1 1 521 257 @b
-1 -1 524 329 1 1 523 257 @b
88 345 :M
-.041(c\) Describe the)A
88 363 :M
-.024(probabilisitic)A
88 381 :M
-.088(behaviour of)A
88 399 :M
-.08(channel gates)A
1 G
0 0 1 1 rF
152 399 :M
psb
/wp$gray 1 def
/wp$top 348 def
/wp$left 191 def
/wp$right 252 def
/wp$bottom 402 def
/wp$box {newpath wp$left wp$bottom moveto wp$left wp$top lineto wp$right wp$top lineto wp$right wp$bottom lineto closepath } def
wp$gray setgray wp$box fill
pse
0 0 1 1 rF
196 345 :M
psb
/wp$gray 1 def
/wp$top 348 def
/wp$left 252 def
/wp$right 313 def
/wp$bottom 402 def
/wp$box {newpath wp$left wp$bottom moveto wp$left wp$top lineto wp$right wp$top lineto wp$right wp$bottom lineto closepath } def
wp$gray setgray wp$box fill
pse
0 0 1 1 rF
257 345 :M
psb
/wp$gray 1 def
/wp$top 348 def
/wp$left 313 def
/wp$right 382 def
/wp$bottom 402 def
/wp$box {newpath wp$left wp$bottom moveto wp$left wp$top lineto wp$right wp$top lineto wp$right wp$bottom lineto closepath } def
wp$gray setgray wp$box fill
pse
0 0 1 1 rF
318 345 :M
psb
/wp$gray 1 def
/wp$top 348 def
/wp$left 382 def
/wp$right 450 def
/wp$bottom 402 def
/wp$box {newpath wp$left wp$bottom moveto wp$left wp$top lineto wp$right wp$top lineto wp$right wp$bottom lineto closepath } def
wp$gray setgray wp$box fill
pse
0 0 1 1 rF
387 345 :M
psb
/wp$gray 1 def
/wp$top 348 def
/wp$left 450 def
/wp$right 521 def
/wp$bottom 402 def
/wp$box {newpath wp$left wp$bottom moveto wp$left wp$top lineto wp$right wp$top lineto wp$right wp$bottom lineto closepath } def
wp$gray setgray wp$box fill
pse
0 G
-1 -1 85 330 1 1 84 329 @b
-1 -1 83 330 1 1 82 329 @b
85 330 -1 1 191 329 1 85 329 @a
-1 -1 192 330 1 1 191 329 @b
192 330 -1 1 252 329 1 192 329 @a
-1 -1 253 330 1 1 252 329 @b
253 330 -1 1 313 329 1 253 329 @a
-1 -1 314 330 1 1 313 329 @b
314 330 -1 1 382 329 1 314 329 @a
-1 -1 383 330 1 1 382 329 @b
383 330 -1 1 450 329 1 383 329 @a
-1 -1 451 330 1 1 450 329 @b
451 330 -1 1 521 329 1 451 329 @a
-1 -1 524 330 1 1 523 329 @b
-1 -1 522 330 1 1 521 329 @b
-1 -1 83 402 1 1 82 330 @b
-1 -1 85 402 1 1 84 330 @b
-1 -1 192 402 1 1 191 330 @b
-1 -1 253 402 1 1 252 330 @b
-1 -1 314 402 1 1 313 330 @b
-1 -1 383 402 1 1 382 330 @b
-1 -1 451 402 1 1 450 330 @b
-1 -1 522 402 1 1 521 330 @b
-1 -1 524 402 1 1 523 330 @b
88 418 :M
-.032(d\) Provide)A
88 436 :M
-.036(antibiotic)A
88 454 :M
-.046(prophylaxis for)A
88 472 :M
-.033(patients allergic to)A
88 490 :M
-.036(penicillin)A
1 G
0 0 1 1 rF
133 490 :M
psb
/wp$gray 1 def
/wp$top 421 def
/wp$left 191 def
/wp$right 252 def
/wp$bottom 493 def
/wp$box {newpath wp$left wp$bottom moveto wp$left wp$top lineto wp$right wp$top lineto wp$right wp$bottom lineto closepath } def
wp$gray setgray wp$box fill
pse
0 0 1 1 rF
196 418 :M
psb
/wp$gray 1 def
/wp$top 421 def
/wp$left 252 def
/wp$right 313 def
/wp$bottom 493 def
/wp$box {newpath wp$left wp$bottom moveto wp$left wp$top lineto wp$right wp$top lineto wp$right wp$bottom lineto closepath } def
wp$gray setgray wp$box fill
pse
0 0 1 1 rF
257 418 :M
psb
/wp$gray 1 def
/wp$top 421 def
/wp$left 313 def
/wp$right 382 def
/wp$bottom 493 def
/wp$box {newpath wp$left wp$bottom moveto wp$left wp$top lineto wp$right wp$top lineto wp$right wp$bottom lineto closepath } def
wp$gray setgray wp$box fill
pse
0 0 1 1 rF
318 418 :M
psb
/wp$gray 1 def
/wp$top 421 def
/wp$left 382 def
/wp$right 450 def
/wp$bottom 493 def
/wp$box {newpath wp$left wp$bottom moveto wp$left wp$top lineto wp$right wp$top lineto wp$right wp$bottom lineto closepath } def
wp$gray setgray wp$box fill
pse
0 0 1 1 rF
387 418 :M
psb
/wp$gray 1 def
/wp$top 421 def
/wp$left 450 def
/wp$right 521 def
/wp$bottom 493 def
/wp$box {newpath wp$left wp$bottom moveto wp$left wp$top lineto wp$right wp$top lineto wp$right wp$bottom lineto closepath } def
wp$gray setgray wp$box fill
pse
0 G
-1 -1 85 403 1 1 84 402 @b
-1 -1 83 403 1 1 82 402 @b
85 403 -1 1 191 402 1 85 402 @a
-1 -1 192 403 1 1 191 402 @b
192 403 -1 1 252 402 1 192 402 @a
-1 -1 253 403 1 1 252 402 @b
253 403 -1 1 313 402 1 253 402 @a
-1 -1 314 403 1 1 313 402 @b
314 403 -1 1 382 402 1 314 402 @a
-1 -1 383 403 1 1 382 402 @b
383 403 -1 1 450 402 1 383 402 @a
-1 -1 451 403 1 1 450 402 @b
451 403 -1 1 521 402 1 451 402 @a
-1 -1 524 403 1 1 523 402 @b
-1 -1 522 403 1 1 521 402 @b
-1 -1 83 493 1 1 82 403 @b
-1 -1 85 493 1 1 84 403 @b
-1 -1 192 493 1 1 191 403 @b
-1 -1 253 493 1 1 252 403 @b
-1 -1 314 493 1 1 313 403 @b
-1 -1 383 493 1 1 382 403 @b
-1 -1 451 493 1 1 450 403 @b
-1 -1 522 493 1 1 521 403 @b
-1 -1 524 493 1 1 523 403 @b
106 545 :M
-.014(These statements are from four different exercises evaluating:)A
106 563 :M
-.002(\(a\) an authoring software tutorial for HE teachers learning to produce simple)A
124 581 :M
-.014(multimedia courseware for their students.)A
106 599 :M
-.002(\(b\) from a study skills package for first year undergraduates, on library search)A
124 617 :M
-.031(strategies.)A
106 635 :M
-.007(\(c\) simulation courseware provided to 3rd year Life Science students during a)A
124 653 :M
-.004(Practical Lab course in Animal Physiology and Neurobiology.)A
106 671 :M
-.002(\(d\) from an evaluation of different teaching components \(tutorials, demonstrations,)A
124 689 :M
-.007(practical work and courseware\) within a 3rd year practical course in)A
124 707 :M
-.023(periodontology.)A
-1 -1 85 494 1 1 84 493 @b
-1 -1 83 494 1 1 82 493 @b
85 494 -1 1 191 493 1 85 493 @a
-1 -1 192 494 1 1 191 493 @b
192 494 -1 1 252 493 1 192 493 @a
-1 -1 253 494 1 1 252 493 @b
253 494 -1 1 313 493 1 253 493 @a
-1 -1 314 494 1 1 313 493 @b
314 494 -1 1 382 493 1 314 493 @a
-1 -1 383 494 1 1 382 493 @b
383 494 -1 1 450 493 1 383 493 @a
-1 -1 451 494 1 1 450 493 @b
451 494 -1 1 521 493 1 451 493 @a
-1 -1 524 494 1 1 523 493 @b
-1 -1 522 494 1 1 521 493 @b
-1 -1 83 728 1 1 82 494 @b
-1 -1 85 728 1 1 84 494 @b
82 731 -1 1 85 730 1 82 730 @a
-1 -1 83 731 1 1 82 728 @b
-1 -1 85 729 1 1 84 728 @b
-1 -1 85 729 1 1 84 728 @b
85 729 -1 1 521 728 1 85 728 @a
85 731 -1 1 521 730 1 85 730 @a
-1 -1 522 728 1 1 521 494 @b
-1 -1 524 728 1 1 523 494 @b
521 731 -1 1 524 730 1 521 730 @a
-1 -1 524 731 1 1 523 728 @b
-1 -1 522 729 1 1 521 728 @b
-1 -1 522 729 1 1 521 728 @b
endp
%%Page: 28 28
%%BeginPageSetup
initializepage
(steve; page: 28 of 29)setjob
%%EndPageSetup
-31 -31 :T
gS 31 31 552 730 rC
36 44 :M
f0_9 sf
( )S
297 44 :M
( )S
549 44 :M
f1_9 sf
(28)S
221 81 :M
f1_12 sf
-.022(Appendix B: Knowledge quizzes)A
36 105 :M
f0_12 sf
-.005(The following example questions, provided by Maggie Pollock, were used during the evaluation of TILT)A
541 105 :M
(-)S
36 117 :M
-.015(funded courseware on the principles of Impulse and Momentum.)A
36 167 :M
(1.)S
72 167 :M
-.013(Momentum is defined as:-)A
324 169 :M
f0_18 sf
( )S
360 167 :M
f0_12 sf
-.12(mv)A
0 -3 rm
(2)S
0 3 rm
324 193 :M
f5_18 sf
( )S
360 193 :M
f0_12 sf
-.146(m)A
0 -3 rm
-.094(2)A
0 3 rm
(v)S
324 217 :M
f5_18 sf
( )S
360 217 :M
f0_12 sf
-.334(mv)A
324 241 :M
f5_18 sf
( )S
360 241 :M
f0_12 sf
-.167(2mv)A
324 265 :M
f5_18 sf
( )S
360 265 :M
f0_12 sf
-.014(I don't know)A
36 325 :M
(3.)S
72 325 :M
-.013(The Impulse of a Force is defined to be the:-)A
324 325 :M
f5_18 sf
( )S
360 325 :M
f0_12 sf
-.018(Change in Velocity)A
324 349 :M
f5_18 sf
( )S
360 349 :M
f0_12 sf
-.058(Change in Momentum)A
324 373 :M
f5_18 sf
( )S
360 373 :M
f0_12 sf
-.042(Change in Kinetic Energy)A
324 397 :M
f5_18 sf
( )S
360 397 :M
f0_12 sf
-.024(Change in Mass)A
324 421 :M
f5_18 sf
( )S
360 421 :M
f0_12 sf
-.014(I don't know)A
36 469 :M
(8.)S
72 469 :M
-.018(If a collision is perfectly elastic, which of the )A
324 469 :M
f6_18 sf
( )S
360 469 :M
f0_12 sf
-.029(The bodies separate after impact)A
72 481 :M
-.031(following statements are true:-)A
324 493 :M
f5_18 sf
( )S
360 493 :M
f0_12 sf
-.031(The bodies stick together after)A
360 505 :M
-.131(impact)A
324 529 :M
f5_18 sf
( )S
360 529 :M
f0_12 sf
-.049(The value of e=0)A
324 553 :M
f5_18 sf
( )S
360 553 :M
f0_12 sf
-.049(The value of e=1)A
324 577 :M
f5_18 sf
( )S
360 577 :M
f0_12 sf
-.053(None of these)A
324 601 :M
f5_18 sf
( )S
360 601 :M
f0_12 sf
-.014(I don't know)A
endp
%%Page: 29 29
%%BeginPageSetup
initializepage
(steve; page: 29 of 29)setjob
%%EndPageSetup
-31 -31 :T
gS 31 31 552 730 rC
36 44 :M
f0_9 sf
( )S
297 44 :M
( )S
549 44 :M
f1_9 sf
(29)S
158 92 :M
f1_18 sf
-.013(Appendix C: Post Task Questionnaire)A
169 117 :M
f2_12 sf
-.018(Excerpt from PTQ used for a language learning package)A
108 168 :M
f1_14 sf
-.01(Course _____________________)A
468 168 :M
f0_14 sf
-.037( )A
f1_12 sf
-.068(Matric. Number)A
-1 -1 475 172 1 1 474 171 @b
-1 -1 475 172 1 1 474 171 @b
475 172 -1 1 488 171 1 475 171 @a
-1 -1 489 172 1 1 488 171 @b
489 172 -1 1 503 171 1 489 171 @a
-1 -1 504 172 1 1 503 171 @b
504 172 -1 1 517 171 1 504 171 @a
-1 -1 518 172 1 1 517 171 @b
518 172 -1 1 531 171 1 518 171 @a
-1 -1 532 172 1 1 531 171 @b
532 172 -1 1 546 171 1 532 171 @a
-1 -1 547 172 1 1 546 171 @b
547 172 -1 1 560 171 1 547 171 @a
-1 -1 561 172 1 1 560 171 @b
561 172 -1 1 575 171 1 561 171 @a
-1 -1 576 172 1 1 575 171 @b
-1 -1 576 172 1 1 575 171 @b
-1 -1 475 184 1 1 474 172 @b
-1 -1 475 185 1 1 474 184 @b
-1 -1 475 185 1 1 474 184 @b
475 185 -1 1 488 184 1 475 184 @a
-1 -1 489 184 1 1 488 172 @b
-1 -1 489 185 1 1 488 184 @b
489 185 -1 1 503 184 1 489 184 @a
-1 -1 504 184 1 1 503 172 @b
-1 -1 504 185 1 1 503 184 @b
504 185 -1 1 517 184 1 504 184 @a
-1 -1 518 184 1 1 517 172 @b
-1 -1 518 185 1 1 517 184 @b
518 185 -1 1 531 184 1 518 184 @a
-1 -1 532 184 1 1 531 172 @b
-1 -1 532 185 1 1 531 184 @b
532 185 -1 1 546 184 1 532 184 @a
-1 -1 547 184 1 1 546 172 @b
-1 -1 547 185 1 1 546 184 @b
547 185 -1 1 560 184 1 547 184 @a
-1 -1 561 184 1 1 560 172 @b
-1 -1 561 185 1 1 560 184 @b
561 185 -1 1 575 184 1 561 184 @a
-1 -1 576 184 1 1 575 172 @b
-1 -1 576 185 1 1 575 184 @b
-1 -1 576 185 1 1 575 184 @b
1 G
0 0 1 1 rF
575 184 :M
psb
/wp$gray 0.9000 def
/wp$top 198 def
/wp$left 34 def
/wp$right 578 def
/wp$bottom 210 def
/wp$box {newpath wp$left wp$bottom moveto wp$left wp$top lineto wp$right wp$top lineto wp$right wp$bottom lineto closepath } def
wp$gray setgray wp$box fill
pse
36 207 :M
0 G
-.01( Read each question carefully. Answer all relevant questions as accurately as you can.)A
-1 -1 34 198 1 1 33 197 @b
-1 -1 34 198 1 1 33 197 @b
34 198 -1 1 578 197 1 34 197 @a
-1 -1 579 198 1 1 578 197 @b
-1 -1 579 198 1 1 578 197 @b
-1 -1 34 210 1 1 33 198 @b
-1 -1 579 210 1 1 578 198 @b
1 G
0 0 1 1 rF
578 198 :M
psb
/wp$gray 0.9000 def
/wp$top 210 def
/wp$left 34 def
/wp$right 578 def
/wp$bottom 222 def
/wp$box {newpath wp$left wp$bottom moveto wp$left wp$top lineto wp$right wp$top lineto wp$right wp$bottom lineto closepath } def
wp$gray setgray wp$box fill
pse
36 219 :M
0 G
-.027( Please PRINT your written answers.)A
-1 -1 34 223 1 1 33 222 @b
-1 -1 34 223 1 1 33 222 @b
34 223 -1 1 578 222 1 34 222 @a
-1 -1 579 223 1 1 578 222 @b
-1 -1 579 223 1 1 578 222 @b
-1 -1 34 222 1 1 33 210 @b
-1 -1 579 222 1 1 578 210 @b
36 256 :M
-.331(2\) )A
52 256 :M
f0_12 sf
-.002( Please list the things you particularly liked and you particularly disliked about the package:-)A
36 280 :M
f1_12 sf
-.2(Liked:)A
36 316 :M
-.124(Disliked:)A
36 352 :M
-.993(3\))A
46 352 :M
f0_12 sf
-.004( What did you think you were meant to learn from this package?)A
36 363 :M
-.036( )A
f2_9 sf
-.045(Please list below:)A
36 422 :M
f1_12 sf
-.331(4\) )A
52 422 :M
f0_12 sf
-.046(Did you?)A
144 422 :M
-.219(Yes )A
166 422 :M
f5_18 sf
( )S
180 422 :M
f0_14 sf
-.06( )A
f0_12 sf
-.151(No )A
219 422 :M
f5_18 sf
( )S
288 422 :M
f0_12 sf
-.029(Some of them)A
36 433 :M
-.009( )A
f2_9 sf
-.011(Please comment further if you wish to:)A
36 468 :M
f1_12 sf
-.331(5\) )A
52 468 :M
f0_12 sf
-.004(Did you feel that the work you did in this Computer package episode)A
432 468 :M
-.219(Yes )A
454 468 :M
f5_18 sf
-.05( )A
540 468 :M
f0_12 sf
-.331(No )A
558 468 :M
f5_18 sf
( )S
36 480 :M
f1_12 sf
( )S
f0_12 sf
-.011( complemented what you covered in this week's language class?)A
36 491 :M
f1_12 sf
-.009( )A
f2_12 sf
( )S
f2_9 sf
-.011(Please comment further if you wish to:)A
36 538 :M
f1_12 sf
-.497(6\) )A
49 538 :M
f0_12 sf
-.012(What did you spend most of your time trying to do during this computer session?)A
36 550 :M
-.014( Please indicate by circling a point on the following scale:)A
98 575 :M
f0_10 sf
-.014(Discovering details of)A
105 586 :M
-.008(how to operate the)A
125 597 :M
-.145(program)A
217 587 :M
f1_12 sf
(\245)S
f0_12 sf
-.033( )A
f1_12 sf
(\245)S
f0_12 sf
-.033( )A
f1_12 sf
(\245)S
272 587 :M
f0_12 sf
-.029( 50/50 )A
f1_12 sf
(\245)S
f0_12 sf
-.024( )A
f1_12 sf
(\245)S
f0_12 sf
-.024( )A
f1_12 sf
(\245)S
1 G
0 0 1 1 rF
389 587 :M
psb
/wp$gray 1 def
/wp$top 590 def
/wp$left 195 def
/wp$right 411 def
/wp$bottom 599 def
/wp$box {newpath wp$left wp$bottom moveto wp$left wp$top lineto wp$right wp$top lineto wp$right wp$bottom lineto closepath } def
wp$gray setgray wp$box fill
pse
431 575 :M
0 G
f0_10 sf
-.01(Concentrating on)A
416 586 :M
-.024(subject-related problems)A
441 597 :M
-.014(and answers)A
.5 G
-1 -1 88 566 1 1 87 565 @b
-1 -1 88 566 1 1 87 565 @b
88 566 -1 1 195 565 1 88 565 @a
-1 -1 196 566 1 1 195 565 @b
-1 -1 196 566 1 1 195 565 @b
-1 -1 412 566 1 1 411 565 @b
-1 -1 412 566 1 1 411 565 @b
412 566 -1 1 519 565 1 412 565 @a
-1 -1 520 566 1 1 519 565 @b
-1 -1 520 566 1 1 519 565 @b
-1 -1 88 599 1 1 87 566 @b
-1 -1 88 600 1 1 87 599 @b
-1 -1 88 600 1 1 87 599 @b
88 600 -1 1 195 599 1 88 599 @a
-1 -1 196 599 1 1 195 566 @b
-1 -1 196 600 1 1 195 599 @b
-1 -1 196 600 1 1 195 599 @b
-1 -1 412 599 1 1 411 566 @b
-1 -1 412 600 1 1 411 599 @b
-1 -1 412 600 1 1 411 599 @b
412 600 -1 1 519 599 1 412 599 @a
-1 -1 520 599 1 1 519 566 @b
-1 -1 520 600 1 1 519 599 @b
-1 -1 520 600 1 1 519 599 @b
36 641 :M
0 G
f1_12 sf
-.005(Comments: )A
f0_9 sf
-.003(\(please feel free to make any comments, positive or negative, that you would like to add to this form, about the)A
36 652 :M
-.002( computer package, or the course as a whole, or the way that we are evaluating it - i.e. the questionnaires!\))A
endp
%%Trailer
end % md
%%EOF