Perhaps the problem was iemEllise vs iemEllipse.  I don't think iemEllise would even compile unless you have it defined somewhere.
This works:
procedure TForm1.MagnifyLayer1Click(Sender: TObject);
var
  iIEMagnifyStyle: TIEMagnifyStyle;
  iMagnifyLayer: Integer;
begin
  iIEMagnifyStyle := iemEllipse;
  with ImageEnVect1 do
  begin
    MouseInteract := MouseInteract + [miMoveLayers, miResizeLayers];
    iMagnifyLayer := LayersAdd;
    LayersCurrent := iMagnifyLayer;
    Layers[iMagnifyLayer].Name := 'Magnifier';
    Layers[iMagnifyLayer].Width := 100;
    Layers[iMagnifyLayer].Height := 100;
    Layers[iMagnifyLayer].PosX := 0;
    Layers[iMagnifyLayer].PosY := 0;
    Layers[iMagnifyLayer].Magnify.Rate := 2;
    Layers[iMagnifyLayer].VisibleBox := True;
    Layers[iMagnifyLayer].DrawOuter := True;
    Layers[iMagnifyLayer].Magnify.Style := iIEMagnifyStyle;
    Layers[iMagnifyLayer].Magnify.Enabled := True;
    Update;
  end;
end;
William Miller
Adirondack Software & Graphics
Email: w2m@frontiernet.net
EBook: http://www.imageen.com/ebook/
Apprehend: http://www.frontiernet.net/~w2m/index.html